From 2f5bd8e63ab497c7c2fb6e0e511c67a8dc4f073d Mon Sep 17 00:00:00 2001 From: azey Date: Wed, 10 Dec 2025 17:25:56 +0100 Subject: [PATCH 1/3] nixos/tests/k3s: generalize to rancher Similarly to 2ce16ee67eae, prepares for merge with nixos/tests/rke2. --- .github/labeler.yml | 2 +- nixos/tests/all-tests.nix | 2 +- nixos/tests/k3s/airgap-images.nix | 34 ----- nixos/tests/k3s/auto-deploy.nix | 124 ------------------ nixos/tests/k3s/default.nix | 34 ----- nixos/tests/rancher/airgap-images.nix | 38 ++++++ .../{k3s => rancher}/auto-deploy-charts.nix | 68 ++++++---- nixos/tests/rancher/auto-deploy.nix | 124 ++++++++++++++++++ .../{k3s => rancher}/containerd-config.nix | 30 ++--- nixos/tests/rancher/default.nix | 66 ++++++++++ nixos/tests/{k3s => rancher}/etcd.nix | 37 +++--- .../tests/{k3s => rancher}/kubelet-config.nix | 26 ++-- nixos/tests/{k3s => rancher}/multi-node.nix | 87 ++++++------ nixos/tests/{k3s => rancher}/single-node.nix | 50 +++---- .../cluster/k3s/docs/ONBOARDING_MAINTAINER.md | 2 +- 15 files changed, 390 insertions(+), 334 deletions(-) delete mode 100644 nixos/tests/k3s/airgap-images.nix delete mode 100644 nixos/tests/k3s/auto-deploy.nix delete mode 100644 nixos/tests/k3s/default.nix create mode 100644 nixos/tests/rancher/airgap-images.nix rename nixos/tests/{k3s => rancher}/auto-deploy-charts.nix (77%) create mode 100644 nixos/tests/rancher/auto-deploy.nix rename nixos/tests/{k3s => rancher}/containerd-config.nix (66%) create mode 100644 nixos/tests/rancher/default.nix rename nixos/tests/{k3s => rancher}/etcd.nix (77%) rename nixos/tests/{k3s => rancher}/kubelet-config.nix (83%) rename nixos/tests/{k3s => rancher}/multi-node.nix (69%) rename nixos/tests/{k3s => rancher}/single-node.nix (67%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 542e327e5026..bfbd131d21a5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -263,7 +263,7 @@ - any-glob-to-any-file: - nixos/modules/services/cluster/rancher/default.nix - nixos/modules/services/cluster/rancher/k3s.nix - - nixos/tests/k3s/**/* + - nixos/tests/rancher/**/* - pkgs/applications/networking/cluster/k3s/**/* "6.topic: kernel": diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index dea336cb1173..98bf23d40206 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -800,7 +800,7 @@ in jitsi-meet = runTest ./jitsi-meet.nix; jool = import ./jool.nix { inherit pkgs runTest; }; jotta-cli = runTest ./jotta-cli.nix; - k3s = handleTest ./k3s { }; + k3s = handleTest ./rancher { rancherDistro = "k3s"; }; kafka = handleTest ./kafka { }; kanboard = runTest ./web-apps/kanboard.nix; kanidm = runTest ./kanidm.nix; diff --git a/nixos/tests/k3s/airgap-images.nix b/nixos/tests/k3s/airgap-images.nix deleted file mode 100644 index 79fd2c77025b..000000000000 --- a/nixos/tests/k3s/airgap-images.nix +++ /dev/null @@ -1,34 +0,0 @@ -# A test that imports k3s airgapped images and verifies that all expected images are present -import ../make-test-python.nix ( - { lib, k3s, ... }: - { - name = "${k3s.name}-airgap-images"; - meta.maintainers = lib.teams.k3s.members; - - nodes.machine = _: { - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - role = "server"; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - ]; - images = [ k3s.airgap-images ]; - }; - }; - - testScript = '' - machine.wait_for_unit("k3s") - machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"") - ''; - } -) diff --git a/nixos/tests/k3s/auto-deploy.nix b/nixos/tests/k3s/auto-deploy.nix deleted file mode 100644 index 6539284b4e3e..000000000000 --- a/nixos/tests/k3s/auto-deploy.nix +++ /dev/null @@ -1,124 +0,0 @@ -# Tests whether container images are imported and auto deploying manifests work -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - pauseImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = pauseImageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - in - { - name = "${k3s.name}-auto-deploy"; - - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = [ k3s ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; - services.k3s.images = [ - pauseImage - helloImage - ]; - services.k3s.manifests = { - absent = { - enable = false; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "absent"; - }; - }; - - present = { - target = "foo-namespace.yaml"; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "foo"; - }; - }; - - hello.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "hello"; - spec = { - template.spec = { - containers = [ - { - name = "hello"; - image = "test.local/hello:local"; - } - ]; - restartPolicy = "OnFailure"; - }; - }; - }; - }; - }; - - testScript = # python - '' - start_all() - - machine.wait_for_unit("k3s") - # check existence of the manifest files - machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml") - - # check if container images got imported - machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") - - # check if resources of manifests got created - machine.wait_until_succeeds("kubectl get ns foo") - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") - machine.fail("kubectl get ns absent") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/default.nix b/nixos/tests/k3s/default.nix deleted file mode 100644 index 8d4203c1fb79..000000000000 --- a/nixos/tests/k3s/default.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allK3s = lib.filterAttrs ( - n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success - ) pkgs; -in -{ - airgap-images = lib.mapAttrs ( - _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; } - ) allK3s; - auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s; - auto-deploy-charts = lib.mapAttrs ( - _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; } - ) allK3s; - containerd-config = lib.mapAttrs ( - _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; } - ) allK3s; - etcd = lib.mapAttrs ( - _: k3s: - import ./etcd.nix { - inherit system pkgs k3s; - inherit (pkgs) etcd; - } - ) allK3s; - kubelet-config = lib.mapAttrs ( - _: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; } - ) allK3s; - multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s; - single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s; -} diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix new file mode 100644 index 000000000000..0cb5bfabdd19 --- /dev/null +++ b/nixos/tests/rancher/airgap-images.nix @@ -0,0 +1,38 @@ +# A test that imports k3s airgapped images and verifies that all expected images are present +import ../make-test-python.nix ( + { + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + ... + }: + { + name = "${rancherPackage.name}-airgap-images"; + meta.maintainers = lib.teams.k3s.members; + + nodes.machine = _: { + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = + { + k3s = [ rancherPackage.airgap-images ]; + } + .${rancherDistro}; + }; + }; + + testScript = '' + machine.wait_for_unit("${serviceName}") + machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") + ''; + } +) diff --git a/nixos/tests/k3s/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix similarity index 77% rename from nixos/tests/k3s/auto-deploy-charts.nix rename to nixos/tests/rancher/auto-deploy-charts.nix index b5d1a31f3c60..848a88c345b9 100644 --- a/nixos/tests/k3s/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -1,15 +1,18 @@ # Tests whether container images are imported and auto deploying Helm charts, -# including the bundled traefik, work +# including the bundled traefik or ingress-nginx, work import ../make-test-python.nix ( { - k3s, - lib, pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let testImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ busybox hello @@ -24,11 +27,11 @@ import ../make-test-python.nix ( }; # pack the test helm chart as a .tgz archive package = - pkgs.runCommand "k3s-test-chart.tgz" + pkgs.runCommand "${rancherDistro}-test-chart.tgz" { nativeBuildInputs = [ pkgs.kubernetes-helm ]; chart = builtins.toJSON { - name = "k3s-test-chart"; + name = "${rancherDistro}-test-chart"; version = "0.1.0"; }; values = builtins.toJSON { @@ -93,7 +96,7 @@ import ../make-test-python.nix ( }; in { - name = "${k3s.name}-auto-deploy-helm"; + name = "${rancherPackage.name}-auto-deploy-helm"; meta.maintainers = lib.teams.k3s.members; nodes.machine = { pkgs, ... }: @@ -104,19 +107,23 @@ import ../make-test-python.nix ( diskSize = 4096; }; environment.systemPackages = [ pkgs.yq-go ]; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - ]; + package = rancherPackage; + disable = + { + k3s = lib.remove "traefik" disabledComponents; + rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; + } + .${rancherDistro}; images = [ - # Provides the k3s Helm controller - k3s.airgap-images + { + # Provides the k3s Helm controller + k3s = rancherPackage.airgap-images; + rke2 = rancherPackage.images-core-linux-amd64-tar-zst; + } + .${rancherDistro} + testImage ]; autoDeployCharts = { @@ -133,7 +140,7 @@ import ../make-test-python.nix ( values = /. + builtins.unsafeDiscardStringContext ( - builtins.toFile "k3s-test-chart-values.yaml" '' + builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' runCommand: "echo 'Hello, file!'" image: repository: test.local/test @@ -176,14 +183,14 @@ import ../make-test-python.nix ( '' import json - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml") - machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml") # check that the timeout is set correctly, select only the first doc in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml")) + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")) t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") # wait for test jobs to complete machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) @@ -197,8 +204,15 @@ import ../make-test-python.nix ( t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") - # wait for bundled traefik deployment - machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + # wait for bundled ingress deployment + ${ + { + k3s = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + ''; + } + .${rancherDistro} + } ''; } ) diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix new file mode 100644 index 000000000000..74784f159cc9 --- /dev/null +++ b/nixos/tests/rancher/auto-deploy.nix @@ -0,0 +1,124 @@ +# Tests whether container images are imported and auto deploying manifests work +import ../make-test-python.nix ( + { + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + ... + }: + let + pauseImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = pauseImageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + helloImage = pkgs.dockerTools.buildImage { + name = "test.local/hello"; + tag = "local"; + copyToRoot = pkgs.hello; + config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; + }; + in + { + name = "${rancherPackage.name}-auto-deploy"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = [ rancherPackage ]; + + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + images = [ + pauseImage + helloImage + ]; + manifests = { + absent = { + enable = false; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "absent"; + }; + }; + + present = { + target = "foo-namespace.yaml"; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "foo"; + }; + }; + + hello.content = { + apiVersion = "batch/v1"; + kind = "Job"; + metadata.name = "hello"; + spec = { + template.spec = { + containers = [ + { + name = "hello"; + image = "test.local/hello:local"; + } + ]; + restartPolicy = "OnFailure"; + }; + }; + }; + }; + }; + }; + + testScript = # python + '' + start_all() + + machine.wait_for_unit("${serviceName}") + # check existence of the manifest files + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + + # check if container images got imported + machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") + + # check if resources of manifests got created + machine.wait_until_succeeds("kubectl get ns foo") + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") + machine.fail("kubectl get ns absent") + ''; + + meta.maintainers = lib.teams.k3s.members; + } +) diff --git a/nixos/tests/k3s/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix similarity index 66% rename from nixos/tests/k3s/containerd-config.nix rename to nixos/tests/rancher/containerd-config.nix index 0ebfd4dac347..a4b064edad21 100644 --- a/nixos/tests/k3s/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -3,14 +3,17 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let nodeName = "test"; in { - name = "${k3s.name}-containerd-config"; + name = "${rancherPackage.name}-containerd-config"; nodes.machine = { ... }: { @@ -19,20 +22,13 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; + package = rancherPackage; + disable = disabledComponents; + inherit nodeName; containerdConfigTemplate = '' - # Base K3s config + # Base ${rancherDistro} config {{ template "base" . }} # MAGIC COMMENT @@ -43,14 +39,14 @@ import ../make-test-python.nix ( testScript = # python '' start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the config template file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl") + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") # test whether the config file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml") + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") ''; diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix new file mode 100644 index 000000000000..89586b517bfd --- /dev/null +++ b/nixos/tests/rancher/default.nix @@ -0,0 +1,66 @@ +{ + system ? builtins.currentSystem, + pkgs ? import ../../.. { inherit system; }, + lib ? pkgs.lib, + # service/package name to test + rancherDistro, +}: +let + allPackages = lib.filterAttrs ( + n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success + ) pkgs; + + mkTestArgs = rancherPackage: { + inherit + system + pkgs + rancherDistro + rancherPackage + ; + + # systemd service name + serviceName = + { + k3s = "k3s"; + rke2 = "rke2-server"; + } + .${rancherDistro}; + + # list passed to services.*.disable, + # for slightly reduced resource usage + disabledComponents = + { + k3s = [ + "coredns" + "local-storage" + "metrics-server" + "servicelb" + "traefik" + ]; + rke2 = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" + ]; + } + .${rancherDistro}; + }; + + importTest = + path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages; +in +{ + airgap-images = importTest ./airgap-images.nix { }; + auto-deploy = importTest ./auto-deploy.nix { }; + auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; + containerd-config = importTest ./containerd-config.nix { }; + etcd = importTest ./etcd.nix { + inherit (pkgs) etcd; + }; + kubelet-config = importTest ./kubelet-config.nix { }; + multi-node = importTest ./multi-node.nix { }; + single-node = importTest ./single-node.nix { }; +} diff --git a/nixos/tests/k3s/etcd.nix b/nixos/tests/rancher/etcd.nix similarity index 77% rename from nixos/tests/k3s/etcd.nix rename to nixos/tests/rancher/etcd.nix index a44de1f6233f..2e40472d4911 100644 --- a/nixos/tests/k3s/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -3,13 +3,16 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, etcd, ... }: { - name = "${k3s.name}-etcd"; + name = "${rancherPackage.name}-etcd"; nodes = { @@ -39,7 +42,7 @@ import ../make-test-python.nix ( }; }; - k3s = + server = { pkgs, ... }: { environment.systemPackages = with pkgs; [ jq ]; @@ -47,18 +50,14 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; role = "server"; - package = k3s; + package = rancherPackage; + disable = disabledComponents; + nodeIP = "192.168.1.2"; extraFlags = [ "--datastore-endpoint=\"http://192.168.1.1:2379\"" - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-ip 192.168.1.2" ]; }; @@ -95,27 +94,27 @@ import ../make-test-python.nix ( with subtest("should wait for etcdctl endpoint health to succeed"): etcd.wait_until_succeeds("etcdctl endpoint health") - with subtest("should start k3s"): - k3s.start() - k3s.wait_for_unit("k3s") + with subtest("should start ${rancherDistro}"): + server.start() + server.wait_for_unit("${serviceName}") with subtest("should test if kubectl works"): - k3s.wait_until_succeeds("k3s kubectl get node") + server.wait_until_succeeds("kubectl get node") with subtest("should wait for service account to show up; takes a sec"): - k3s.wait_until_succeeds("k3s kubectl get serviceaccount default") + server.wait_until_succeeds("kubectl get serviceaccount default") with subtest("should create a sample secret object"): - k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") with subtest("should check if secret is correct"): - k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") with subtest("should have a secret in database"): etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") with subtest("should delete the secret"): - k3s.succeed("k3s kubectl delete secret nixossecret") + server.succeed("kubectl delete secret nixossecret") with subtest("should not have a secret in database"): etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") diff --git a/nixos/tests/k3s/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix similarity index 83% rename from nixos/tests/k3s/kubelet-config.nix rename to nixos/tests/rancher/kubelet-config.nix index f5aacd22a13f..322d40e69108 100644 --- a/nixos/tests/k3s/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -3,7 +3,10 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let @@ -15,7 +18,7 @@ import ../make-test-python.nix ( containerLogMaxSize = "5Mi"; in { - name = "${k3s.name}-kubelet-config"; + name = "${rancherPackage.name}-kubelet-config"; nodes.machine = { pkgs, ... }: { @@ -25,18 +28,11 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; + package = rancherPackage; + disable = disabledComponents; + inherit nodeName; gracefulNodeShutdown = { enable = true; inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; @@ -52,11 +48,11 @@ import ../make-test-python.nix ( import json start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"") + machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"") # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") machine.wait_until_succeeds("nc -z 127.0.0.1 8001") diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/rancher/multi-node.nix similarity index 69% rename from nixos/tests/k3s/multi-node.nix rename to nixos/tests/rancher/multi-node.nix index fdf825a3c103..14671a7ae59c 100644 --- a/nixos/tests/k3s/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -1,14 +1,17 @@ -# A test that runs a multi-node k3s cluster and verify pod networking works across nodes +# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ tini bashInteractive @@ -56,11 +59,16 @@ import ../make-test-python.nix ( tokenFile = pkgs.writeText "token" "p@s$w0rd"; in { - name = "${k3s.name}-multi-node"; + name = "${rancherPackage.name}-multi-node"; nodes = { server = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { environment.systemPackages = with pkgs; [ gzip @@ -70,21 +78,17 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "server"; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; clusterInit = true; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" "--pause-image test.local/pause:local" - "--node-ip ${nodes.server.networking.primaryIPAddress}" # The interface selection logic of flannel would normally use eth0, as the nixos # testing driver sets a default route via dev eth0. However, in test setups we # have to use eth1 for inter-node communication. @@ -100,7 +104,12 @@ import ../make-test-python.nix ( }; server2 = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { environment.systemPackages = with pkgs; [ gzip @@ -109,21 +118,17 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443"; clusterInit = false; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" "--pause-image test.local/pause:local" - "--node-ip ${nodes.server2.networking.primaryIPAddress}" "--flannel-iface eth1" ]; }; @@ -136,20 +141,25 @@ import ../make-test-python.nix ( }; agent = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { virtualisation.memorySize = 1024; virtualisation.diskSize = 2048; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "agent"; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443"; + nodeIP = config.networking.primaryIPAddress; extraFlags = [ "--pause-image test.local/pause:local" - "--node-ip ${nodes.agent.networking.primaryIPAddress}" "--flannel-iface eth1" ]; }; @@ -164,25 +174,28 @@ import ../make-test-python.nix ( machines = [server, server2, agent] for m in machines: - m.wait_for_unit("k3s") + m.wait_for_unit("${serviceName}") # wait for the agent to show up - server.wait_until_succeeds("k3s kubectl get node agent") + server.wait_until_succeeds("kubectl get node agent") + server.succeed("kubectl get node >&2") - for m in machines: - m.succeed("k3s check-config") + ${lib.optionalString (rancherDistro == "k3s") '' + for m in machines: + m.succeed("k3s check-config") + ''} - server.succeed("k3s kubectl cluster-info") + server.succeed("kubectl cluster-info") # Also wait for our service account to show up; it takes a sec - server.wait_until_succeeds("k3s kubectl get serviceaccount default") + server.wait_until_succeeds("kubectl get serviceaccount default") # Now create a pod on each node via a daemonset and verify they can talk to each other. - server.succeed("k3s kubectl apply -f ${networkTestDaemonset}") - server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') + server.succeed("kubectl apply -f ${networkTestDaemonset}") + server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') # Get pod IPs - pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() - pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] + pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() + pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] # Verify each server can ping each pod ip for pod_ip in pod_ips: @@ -191,7 +204,7 @@ import ../make-test-python.nix ( agent.succeed(f"ping -c 1 {pod_ip}") # Verify the pods can talk to each other for pod in pods: - resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") + resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") t.assertEqual(resp.strip(), "server") ''; diff --git a/nixos/tests/k3s/single-node.nix b/nixos/tests/rancher/single-node.nix similarity index 67% rename from nixos/tests/k3s/single-node.nix rename to nixos/tests/rancher/single-node.nix index 0a6f688afddc..d894b951d798 100644 --- a/nixos/tests/k3s/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -1,14 +1,17 @@ -# A test that runs a single node k3s cluster and verify a pod can run +# A test that runs a single node rancher cluster and verifies a pod can run import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ tini (lib.hiPrio coreutils) @@ -40,13 +43,13 @@ import ../make-test-python.nix ( ''; in { - name = "${k3s.name}-single-node"; + name = "${rancherPackage.name}-single-node"; nodes.machine = { pkgs, ... }: { environment.systemPackages = with pkgs; [ - k3s + rancherPackage gzip ]; @@ -54,23 +57,20 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + }; users.users = { noprivs = { isNormalUser = true; - description = "Can't access k3s by default"; + description = "Can't access ${rancherDistro} by default"; password = "*"; }; }; @@ -80,10 +80,12 @@ import ../make-test-python.nix ( '' start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") machine.succeed("kubectl cluster-info") machine.fail("sudo -u noprivs kubectl cluster-info") - machine.succeed("k3s check-config") + ${lib.optionalString (rancherDistro == "k3s") '' + machine.succeed("k3s check-config") + ''} machine.succeed( "${pauseImage} | ctr image import -" ) @@ -95,16 +97,16 @@ import ../make-test-python.nix ( machine.succeed("kubectl delete -f ${testPodYaml}") # regression test for #176445 - machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'") + machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") - with subtest("Run k3s-killall"): + with subtest("Run ${rancherDistro}-killall"): # Call the killall script with a clean path to assert that # all required commands are wrapped - output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr") + output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") t.assertNotIn("command not found", output, "killall script contains unknown command") # Check that killall cleaned up properly - machine.fail("systemctl is-active k3s.service") + machine.fail("systemctl is-active ${serviceName}.service") machine.fail("systemctl list-units | grep containerd") machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") machine.fail("ip netns show | grep cni-") diff --git a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md index 580c614bbee3..23eff54b0828 100644 --- a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md +++ b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md @@ -7,7 +7,7 @@ A K3s maintainer, maintains K3s's: - [documentation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/README.md) - [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s) - [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22) -- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s) +- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/rancher) - [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher) - [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating) - updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/) From e963fce108f2981e300f5dca663a15b4b1b6f96a Mon Sep 17 00:00:00 2001 From: azey Date: Thu, 11 Dec 2025 01:10:30 +0100 Subject: [PATCH 2/3] nixos/tests/rke2: merge into nixos/tests/k3s Follow-up on ac4abf7f7ba9. --- nixos/tests/all-tests.nix | 2 +- nixos/tests/rancher/airgap-images.nix | 14 +- nixos/tests/rancher/auto-deploy-charts.nix | 60 +++--- nixos/tests/rancher/auto-deploy.nix | 36 ++-- nixos/tests/rancher/containerd-config.nix | 16 +- nixos/tests/rancher/default.nix | 41 +++- nixos/tests/rancher/etcd.nix | 18 +- nixos/tests/rancher/kubelet-config.nix | 17 +- nixos/tests/rancher/multi-node.nix | 123 +++++++----- nixos/tests/rancher/single-node.nix | 25 ++- nixos/tests/rke2/default.nix | 14 -- nixos/tests/rke2/multi-node.nix | 207 --------------------- nixos/tests/rke2/single-node.nix | 144 -------------- 13 files changed, 236 insertions(+), 481 deletions(-) delete mode 100644 nixos/tests/rke2/default.nix delete mode 100644 nixos/tests/rke2/multi-node.nix delete mode 100644 nixos/tests/rke2/single-node.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 98bf23d40206..4859fd3b9df6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1346,7 +1346,7 @@ in restic-rest-server = runTest ./restic-rest-server.nix; retroarch = runTest ./retroarch.nix; ringboard = runTest ./ringboard.nix; - rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { }; + rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; }; rkvm = handleTest ./rkvm { }; rmfakecloud = runTest ./rmfakecloud.nix; robustirc-bridge = runTest ./robustirc-bridge.nix; diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix index 0cb5bfabdd19..cfc6e8da7f13 100644 --- a/nixos/tests/rancher/airgap-images.nix +++ b/nixos/tests/rancher/airgap-images.nix @@ -1,21 +1,21 @@ # A test that imports k3s airgapped images and verifies that all expected images are present import ../make-test-python.nix ( { + pkgs, lib, rancherDistro, rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: { name = "${rancherPackage.name}-airgap-images"; - meta.maintainers = lib.teams.k3s.members; nodes.machine = _: { - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; @@ -23,8 +23,10 @@ import ../make-test-python.nix ( package = rancherPackage; disable = disabledComponents; images = - { + coreImages + ++ { k3s = [ rancherPackage.airgap-images ]; + rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages } .${rancherDistro}; }; @@ -34,5 +36,7 @@ import ../make-test-python.nix ( machine.wait_for_unit("${serviceName}") machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix index 848a88c345b9..e0a954b915f6 100644 --- a/nixos/tests/rancher/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -8,6 +8,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -97,16 +99,18 @@ import ../make-test-python.nix ( in { name = "${rancherPackage.name}-auto-deploy-helm"; - meta.maintainers = lib.teams.k3s.members; + nodes.machine = { pkgs, ... }: { - # k3s uses enough resources the default vm fails. - virtualisation = { - memorySize = 1536; - diskSize = 4096; - }; - environment.systemPackages = [ pkgs.yq-go ]; + environment.systemPackages = with pkgs; [ + kubectl + yq-go + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + services.${rancherDistro} = { enable = true; package = rancherPackage; @@ -116,16 +120,13 @@ import ../make-test-python.nix ( rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; } .${rancherDistro}; - images = [ - { - # Provides the k3s Helm controller - k3s = rancherPackage.airgap-images; - rke2 = rancherPackage.images-core-linux-amd64-tar-zst; - } - .${rancherDistro} - - testImage - ]; + images = + coreImages + # Provides the k3s Helm controller + ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images + ++ [ + testImage + ]; autoDeployCharts = { # regular test chart that should get installed hello = testChart; @@ -180,17 +181,25 @@ import ../make-test-python.nix ( }; testScript = # python + let + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; + in '' import json machine.wait_for_unit("${serviceName}") # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") - machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml") - # check that the timeout is set correctly, select only the first doc in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")) + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") + # check that the timeout is set correctly, select only the first item in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") # wait for test jobs to complete machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) @@ -210,9 +219,14 @@ import ../make-test-python.nix ( k3s = '' machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) ''; + rke2 = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) + ''; } .${rancherDistro} } ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix index 74784f159cc9..5d3abfff040f 100644 --- a/nixos/tests/rancher/auto-deploy.nix +++ b/nixos/tests/rancher/auto-deploy.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -35,6 +37,13 @@ import ../make-test-python.nix ( copyToRoot = pkgs.hello; config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; }; + + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; in { name = "${rancherPackage.name}-auto-deploy"; @@ -42,11 +51,13 @@ import ../make-test-python.nix ( nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ rancherPackage ]; + environment.systemPackages = with pkgs; [ + kubectl + cri-tools + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; @@ -56,7 +67,7 @@ import ../make-test-python.nix ( extraFlags = [ "--pause-image test.local/pause:local" ]; - images = [ + images = coreImages ++ [ pauseImage helloImage ]; @@ -71,7 +82,7 @@ import ../make-test-python.nix ( }; present = { - target = "foo-namespace.yaml"; + target = "foo-namespace.${manifestFormat}"; content = { apiVersion = "v1"; kind = "Namespace"; @@ -105,13 +116,14 @@ import ../make-test-python.nix ( machine.wait_for_unit("${serviceName}") # check existence of the manifest files - machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") # check if container images got imported - machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") + # for some reason, RKE2 also uses /run/k3s + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") # check if resources of manifests got created machine.wait_until_succeeds("kubectl get ns foo") @@ -119,6 +131,6 @@ import ../make-test-python.nix ( machine.fail("kubectl get ns absent") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix index a4b064edad21..f7ab7af2e758 100644 --- a/nixos/tests/rancher/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -17,15 +19,19 @@ import ../make-test-python.nix ( nodes.machine = { ... }: { - environment.systemPackages = [ pkgs.jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; services.${rancherDistro} = { enable = true; package = rancherPackage; disable = disabledComponents; + images = coreImages; inherit nodeName; containerdConfigTemplate = '' # Base ${rancherDistro} config @@ -50,6 +56,6 @@ import ../make-test-python.nix ( t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix index 89586b517bfd..2ca5db896e32 100644 --- a/nixos/tests/rancher/default.nix +++ b/nixos/tests/rancher/default.nix @@ -47,6 +47,43 @@ let ]; } .${rancherDistro}; + + # images that must be present for all tests + coreImages = + { + k3s = [ ]; + + rke2 = + { + aarch64-linux = [ + rancherPackage.images-core-linux-arm64-tar-zst + rancherPackage.images-canal-linux-arm64-tar-zst + ]; + x86_64-linux = [ + rancherPackage.images-core-linux-amd64-tar-zst + rancherPackage.images-canal-linux-amd64-tar-zst + ]; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); + } + .${rancherDistro}; + + # virtualization.* attrs, since all distros + # need more resources than the default + vmResources = + { + k3s = { + memorySize = 1536; + diskSize = 4096; + }; + rke2 = { + cores = 4; + memorySize = 4096; + diskSize = 8092; + }; + } + .${rancherDistro}; }; importTest = @@ -57,9 +94,7 @@ in auto-deploy = importTest ./auto-deploy.nix { }; auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; containerd-config = importTest ./containerd-config.nix { }; - etcd = importTest ./etcd.nix { - inherit (pkgs) etcd; - }; + etcd = importTest ./etcd.nix { }; kubelet-config = importTest ./kubelet-config.nix { }; multi-node = importTest ./multi-node.nix { }; single-node = importTest ./single-node.nix { }; diff --git a/nixos/tests/rancher/etcd.nix b/nixos/tests/rancher/etcd.nix index 2e40472d4911..0bc1941dd51a 100644 --- a/nixos/tests/rancher/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -7,7 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, - etcd, + coreImages, + vmResources, ... }: @@ -45,16 +46,20 @@ import ../make-test-python.nix ( server = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; services.${rancherDistro} = { enable = true; role = "server"; package = rancherPackage; disable = disabledComponents; + images = coreImages; nodeIP = "192.168.1.2"; extraFlags = [ "--datastore-endpoint=\"http://192.168.1.1:2379\"" @@ -120,6 +125,7 @@ import ../make-test-python.nix ( etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") ''; - meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members; + meta.maintainers = + pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix index 322d40e69108..fb921876a584 100644 --- a/nixos/tests/rancher/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -22,16 +24,19 @@ import ../make-test-python.nix ( nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ pkgs.jq ]; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; package = rancherPackage; disable = disabledComponents; + images = coreImages; inherit nodeName; gracefulNodeShutdown = { enable = true; @@ -52,7 +57,7 @@ import ../make-test-python.nix ( # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"") + machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") machine.wait_until_succeeds("nc -z 127.0.0.1 8001") @@ -67,6 +72,6 @@ import ../make-test-python.nix ( t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/multi-node.nix b/nixos/tests/rancher/multi-node.nix index 14671a7ae59c..acc077613d64 100644 --- a/nixos/tests/rancher/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -57,6 +59,13 @@ import ../make-test-python.nix ( command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] ''; tokenFile = pkgs.writeText "token" "p@s$w0rd"; + + supervisorPort = + { + k3s = "6443"; + rke2 = "9345"; + } + .${rancherDistro}; in { name = "${rancherPackage.name}-multi-node"; @@ -71,34 +80,63 @@ import ../make-test-python.nix ( }: { environment.systemPackages = with pkgs; [ + kubectl gzip jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - services.${rancherDistro} = { - inherit tokenFile; - enable = true; - role = "server"; - package = rancherPackage; - images = [ pauseImage ]; - clusterInit = true; - nodeIP = config.networking.primaryIPAddress; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - # The interface selection logic of flannel would normally use eth0, as the nixos - # testing driver sets a default route via dev eth0. However, in test setups we - # have to use eth1 for inter-node communication. - "--flannel-iface eth1" - ]; - }; + virtualisation = vmResources; + + services.${rancherDistro} = lib.mkMerge [ + { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + } + { + k3s = { + clusterInit = true; + extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition + }; + + # The interface selection logic of flannel & canal would normally use eth0, as + # the nixos testing driver sets a default route via dev eth0. However, in test + # setups we have to use eth1 for inter-node communication. + # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be + # configured with this manifest. + rke2.manifests.canal-config.content = { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChartConfig"; + metadata = { + name = "rke2-canal"; + namespace = "kube-system"; + }; + # spec.valuesContent needs to a string, either json or yaml + spec.valuesContent = builtins.toJSON { + flannel.iface = "eth1"; + }; + }; + } + .${rancherDistro} + ]; + + networking.firewall.enable = false; networking.firewall.allowedTCPPorts = [ 2379 2380 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 ]; networking.firewall.allowedUDPPorts = [ 8472 ]; }; @@ -111,31 +149,32 @@ import ../make-test-python.nix ( ... }: { - environment.systemPackages = with pkgs; [ - gzip - jq - ]; - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { inherit tokenFile; enable = true; + role = "server"; package = rancherPackage; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443"; - clusterInit = false; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; disable = disabledComponents; extraFlags = [ "--pause-image test.local/pause:local" - "--flannel-iface eth1" - ]; + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; + + networking.firewall.enable = false; networking.firewall.allowedTCPPorts = [ 2379 2380 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 ]; networking.firewall.allowedUDPPorts = [ 8472 ]; }; @@ -148,22 +187,23 @@ import ../make-test-python.nix ( ... }: { - virtualisation.memorySize = 1024; - virtualisation.diskSize = 2048; + virtualisation = vmResources; + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "agent"; package = rancherPackage; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443"; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; extraFlags = [ "--pause-image test.local/pause:local" - "--flannel-iface eth1" - ]; + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - networking.firewall.allowedTCPPorts = [ 6443 ]; + + networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; networking.firewall.allowedUDPPorts = [ 8472 ]; }; }; @@ -172,13 +212,12 @@ import ../make-test-python.nix ( '' start_all() - machines = [server, server2, agent] - for m in machines: + servers = [server, server2] + for m in servers: m.wait_for_unit("${serviceName}") # wait for the agent to show up server.wait_until_succeeds("kubectl get node agent") - server.succeed("kubectl get node >&2") ${lib.optionalString (rancherDistro == "k3s") '' for m in machines: @@ -208,6 +247,6 @@ import ../make-test-python.nix ( t.assertEqual(resp.strip(), "server") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/single-node.nix b/nixos/tests/rancher/single-node.nix index d894b951d798..57c1691afdee 100644 --- a/nixos/tests/rancher/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -18,7 +20,7 @@ import ../make-test-python.nix ( busybox ]; }; - pauseImage = pkgs.dockerTools.streamLayeredImage { + pauseImage = pkgs.dockerTools.buildLayeredImage { name = "test.local/pause"; tag = "local"; contents = imageEnv; @@ -29,7 +31,7 @@ import ../make-test-python.nix ( "inf" ]; }; - testPodYaml = pkgs.writeText "test.yml" '' + testPodYaml = pkgs.writeText "test.yaml" '' apiVersion: v1 kind: Pod metadata: @@ -46,22 +48,22 @@ import ../make-test-python.nix ( name = "${rancherPackage.name}-single-node"; nodes.machine = - { pkgs, ... }: + { config, pkgs, ... }: { environment.systemPackages = with pkgs; [ - rancherPackage + kubectl gzip ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; role = "server"; package = rancherPackage; disable = disabledComponents; + images = coreImages ++ [ pauseImage ]; extraFlags = [ "--pause-image test.local/pause:local" ]; @@ -86,14 +88,11 @@ import ../make-test-python.nix ( ${lib.optionalString (rancherDistro == "k3s") '' machine.succeed("k3s check-config") ''} - machine.succeed( - "${pauseImage} | ctr image import -" - ) # Also wait for our service account to show up; it takes a sec machine.wait_until_succeeds("kubectl get serviceaccount default") machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") machine.succeed("kubectl delete -f ${testPodYaml}") # regression test for #176445 @@ -107,11 +106,11 @@ import ../make-test-python.nix ( # Check that killall cleaned up properly machine.fail("systemctl is-active ${serviceName}.service") - machine.fail("systemctl list-units | grep containerd") + machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") machine.fail("ip netns show | grep cni-") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rke2/default.nix b/nixos/tests/rke2/default.nix deleted file mode 100644 index 90415a262acf..000000000000 --- a/nixos/tests/rke2/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs; -in -{ - # Run a single node rke2 cluster and verify a pod can run - singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2; - # Run a multi-node rke2 cluster and verify pod networking works across nodes - multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2; -} diff --git a/nixos/tests/rke2/multi-node.nix b/nixos/tests/rke2/multi-node.nix deleted file mode 100644 index ee394c07f828..000000000000 --- a/nixos/tests/rke2/multi-node.nix +++ /dev/null @@ -1,207 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.buildEnv { - name = "rke2-hello-image-env"; - paths = with pkgs; [ - coreutils - socat - ]; - }; - }; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; - agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd"; - # Let flannel use eth1 to enable inter-node communication in tests - canalConfig = { - apiVersion = "helm.cattle.io/v1"; - kind = "HelmChartConfig"; - metadata = { - name = "rke2-canal"; - namespace = "kube-system"; - }; - # spec.valuesContent needs to a string, either json or yaml - spec.valuesContent = builtins.toJSON { - flannel.iface = "eth1"; - }; - }; - in - { - name = "${rke2.name}-multi-node"; - meta.maintainers = rke2.meta.maintainers; - - nodes = { - server = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.allowedTCPPorts = [ - # Kubernetes API - 6443 - # Canal CNI health checks - 9099 - # RKE2 supervisor API - 9345 - ]; - - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - inherit tokenFile; - inherit agentTokenFile; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - canal-config.content = canalConfig; - # A daemonset that responds 'hello' on port 8000 - network-test.content = { - apiVersion = "apps/v1"; - kind = "DaemonSet"; - metadata = { - name = "test"; - labels.name = "test"; - }; - spec = { - selector.matchLabels.name = "test"; - template = { - metadata.labels.name = "test"; - spec.containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - imagePullPolicy = "Never"; - command = [ - "socat" - "TCP4-LISTEN:8000,fork" - "EXEC:echo hello" - ]; - } - ]; - }; - }; - }; - }; - }; - }; - - agent = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI health checks - networking.firewall.allowedTCPPorts = [ 9099 ]; - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - - # The agent node can work with less resources - virtualisation.memorySize = 2048; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "agent"; - package = rke2; - tokenFile = agentTokenFile; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345"; - nodeIP = config.networking.primaryIPAddress; - manifests.canal-config.content = canalConfig; - images = [ - coreImages - canalImages - helloImage - ]; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - jq = "${pkgs.jq}/bin/jq"; - in - # python - '' - start_all() - - server.wait_for_unit("rke2-server") - agent.wait_for_unit("rke2-agent") - - # Wait for the agent to be ready - server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""") - - server.succeed("${kubectl} cluster-info") - server.wait_until_succeeds("${kubectl} get serviceaccount default") - - # Now verify that each daemonset pod can talk to each other. - server.wait_until_succeeds( - f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]' - ) - - # Get pod IPs - pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines() - pod_ips = [ - server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods - ] - - # Verify each node can ping each pod ip - for pod_ip in pod_ips: - # The CNI sometimes needs a little time - server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - # Verify the server can exec into the pod - for pod in pods: - resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip() - assert resp == "hello", f"Unexpected response from hello daemonset: {resp}" - ''; - } -) diff --git a/nixos/tests/rke2/single-node.nix b/nixos/tests/rke2/single-node.nix deleted file mode 100644 index e19cccf5acc0..000000000000 --- a/nixos/tests/rke2/single-node.nix +++ /dev/null @@ -1,144 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - # A ConfigMap in regular yaml format - cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "from-file"; - data.username = "foo-file"; - }; - in - { - name = "${rke2.name}-single-node"; - meta.maintainers = rke2.meta.maintainers; - nodes.machine = - { - config, - nodes, - pkgs, - ... - }: - { - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - # Slightly reduce resource consumption - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - test-job.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "test"; - spec.template.spec = { - containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - } - ]; - restartPolicy = "Never"; - }; - }; - disabled = { - enable = false; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "disabled"; - data.username = "foo"; - }; - }; - from-file.source = "${cmFile}"; - custom-target = { - enable = true; - target = "my-manifest.json"; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "custom-target"; - data.username = "foo-custom"; - }; - }; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - in - # python - '' - start_all() - - with subtest("Start cluster"): - machine.wait_for_unit("rke2-server") - machine.succeed("${kubectl} cluster-info") - machine.wait_until_succeeds("${kubectl} get serviceaccount default") - - with subtest("Test job completes successfully"): - machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test") - output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip() - assert output == "Hello, world!", f"unexpected output of test job: {output}" - - with subtest("ConfigMap from-file exists"): - output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}" - - with subtest("ConfigMap custom-target exists"): - # Check that the file exists at the custom target path - machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json") - output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}" - - with subtest("Disabled ConfigMap doesn't exist"): - machine.fail("${kubectl} get cm disabled") - ''; - } -) From 49bd3e5ef62ca60cfaf19fd577c33c1adb6d3e6b Mon Sep 17 00:00:00 2001 From: azey Date: Thu, 11 Dec 2025 19:06:45 +0100 Subject: [PATCH 3/3] nixos/tests/{k3s,rke2}: migrate to runTest Also exposed a .all attr alongside existing tests, for easily running all tests for all package versions. --- nixos/tests/all-tests.nix | 16 +- nixos/tests/rancher/airgap-images.nix | 70 ++- nixos/tests/rancher/auto-deploy-charts.nix | 434 +++++++++-------- nixos/tests/rancher/auto-deploy.nix | 234 +++++----- nixos/tests/rancher/containerd-config.nix | 106 +++-- nixos/tests/rancher/default.nix | 183 ++++---- nixos/tests/rancher/etcd.nix | 224 +++++---- nixos/tests/rancher/kubelet-config.nix | 132 +++--- nixos/tests/rancher/multi-node.nix | 436 +++++++++--------- nixos/tests/rancher/single-node.nix | 208 +++++---- .../networking/cluster/k3s/builder.nix | 11 +- .../networking/cluster/rke2/builder.nix | 22 +- 12 files changed, 1039 insertions(+), 1037 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4859fd3b9df6..5f9bed4f1b7e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -800,7 +800,11 @@ in jitsi-meet = runTest ./jitsi-meet.nix; jool = import ./jool.nix { inherit pkgs runTest; }; jotta-cli = runTest ./jotta-cli.nix; - k3s = handleTest ./rancher { rancherDistro = "k3s"; }; + k3s = import ./rancher { + inherit pkgs runTest; + inherit (pkgs) lib; + rancherDistro = "k3s"; + }; kafka = handleTest ./kafka { }; kanboard = runTest ./web-apps/kanboard.nix; kanidm = runTest ./kanidm.nix; @@ -1346,7 +1350,15 @@ in restic-rest-server = runTest ./restic-rest-server.nix; retroarch = runTest ./retroarch.nix; ringboard = runTest ./ringboard.nix; - rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; }; + rke2 = import ./rancher { + inherit pkgs; + inherit (pkgs) lib; + runTest = runTestOn [ + "aarch64-linux" + "x86_64-linux" + ]; + rancherDistro = "rke2"; + }; rkvm = handleTest ./rkvm { }; rmfakecloud = runTest ./rmfakecloud.nix; robustirc-bridge = runTest ./robustirc-bridge.nix; diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix index cfc6e8da7f13..b9dabb5ddbf9 100644 --- a/nixos/tests/rancher/airgap-images.nix +++ b/nixos/tests/rancher/airgap-images.nix @@ -1,42 +1,40 @@ # A test that imports k3s airgapped images and verifies that all expected images are present -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - { - name = "${rancherPackage.name}-airgap-images"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +{ + name = "${rancherPackage.name}-airgap-images"; - nodes.machine = _: { - virtualisation = vmResources; + nodes.machine = _: { + virtualisation = vmResources; - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = - coreImages - ++ { - k3s = [ rancherPackage.airgap-images ]; - rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages - } - .${rancherDistro}; - }; + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = + coreImages + ++ { + k3s = [ rancherPackage.airgap-images ]; + rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages + } + .${rancherDistro}; }; + }; - testScript = '' - machine.wait_for_unit("${serviceName}") - machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") - ''; + testScript = '' + machine.wait_for_unit("${serviceName}") + machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix index e0a954b915f6..125f22790db8 100644 --- a/nixos/tests/rancher/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -1,232 +1,230 @@ # Tests whether container images are imported and auto deploying Helm charts, # including the bundled traefik or ingress-nginx, work -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - testImageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - busybox - hello - ]; - }; - testImage = pkgs.dockerTools.buildImage { - name = "test.local/test"; - tag = "local"; - # Slightly reduces the time needed to import image - compressor = "zstd"; - copyToRoot = testImageEnv; - }; - # pack the test helm chart as a .tgz archive - package = - pkgs.runCommand "${rancherDistro}-test-chart.tgz" - { - nativeBuildInputs = [ pkgs.kubernetes-helm ]; - chart = builtins.toJSON { - name = "${rancherDistro}-test-chart"; - version = "0.1.0"; - }; - values = builtins.toJSON { - restartPolicy = "Never"; - runCommand = ""; - image = { - repository = "foo"; - tag = "1.0.0"; - }; - }; - job = builtins.toJSON { - apiVersion = "batch/v1"; - kind = "Job"; - metadata = { - name = "{{ .Release.Name }}"; - namespace = "{{ .Release.Namespace }}"; - }; - spec = { - template = { - spec = { - containers = [ - { - name = "test"; - image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; - command = [ "sh" ]; - args = [ - "-c" - "{{ .Values.runCommand }}" - ]; - } - ]; - restartPolicy = "{{ .Values.restartPolicy }}"; - }; - }; - }; - }; - passAsFile = [ - "values" - "chart" - "job" - ]; - } - '' - mkdir -p chart/templates - cp "$chartPath" chart/Chart.yaml - cp "$valuesPath" chart/values.yaml - cp "$jobPath" chart/templates/job.json - - helm package chart - mv ./*.tgz $out - ''; - # The common Helm chart that is used in this test - testChart = { - inherit package; - values = { - runCommand = "hello"; - image = { - repository = testImage.imageName; - tag = testImage.imageTag; - }; - }; - }; - in - { - name = "${rancherPackage.name}-auto-deploy-helm"; - - nodes.machine = - { pkgs, ... }: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + testImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + busybox + hello + ]; + }; + testImage = pkgs.dockerTools.buildImage { + name = "test.local/test"; + tag = "local"; + # Slightly reduces the time needed to import image + compressor = "zstd"; + copyToRoot = testImageEnv; + }; + # pack the test helm chart as a .tgz archive + package = + pkgs.runCommand "${rancherDistro}-test-chart.tgz" { - environment.systemPackages = with pkgs; [ - kubectl - yq-go - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - package = rancherPackage; - disable = - { - k3s = lib.remove "traefik" disabledComponents; - rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; - } - .${rancherDistro}; - images = - coreImages - # Provides the k3s Helm controller - ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images - ++ [ - testImage - ]; - autoDeployCharts = { - # regular test chart that should get installed - hello = testChart; - # disabled chart that should not get installed - disabled = testChart // { - enable = false; - }; - # chart with values set via YAML file - values-file = testChart // { - # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path - # https://github.com/NixOS/nix/issues/12407 - values = - /. - + builtins.unsafeDiscardStringContext ( - builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' - runCommand: "echo 'Hello, file!'" - image: - repository: test.local/test - tag: local - '' - ); - }; - # advanced chart that should get installed in the "test" namespace with a custom - # timeout and overridden values - advanced = testChart // { - # create the "test" namespace via extraDeploy for testing - extraDeploy = [ - { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "test"; - } - ]; - extraFieldDefinitions = { - spec = { - # overwrite chart values - valuesContent = '' - runCommand: "echo 'advanced hello'" - image: - repository: ${testImage.imageName} - tag: ${testImage.imageTag} - ''; - # overwrite the chart namespace - targetNamespace = "test"; - # set a custom timeout - timeout = "69s"; - }; + nativeBuildInputs = [ pkgs.kubernetes-helm ]; + chart = builtins.toJSON { + name = "${rancherDistro}-test-chart"; + version = "0.1.0"; + }; + values = builtins.toJSON { + restartPolicy = "Never"; + runCommand = ""; + image = { + repository = "foo"; + tag = "1.0.0"; + }; + }; + job = builtins.toJSON { + apiVersion = "batch/v1"; + kind = "Job"; + metadata = { + name = "{{ .Release.Name }}"; + namespace = "{{ .Release.Namespace }}"; + }; + spec = { + template = { + spec = { + containers = [ + { + name = "test"; + image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; + command = [ "sh" ]; + args = [ + "-c" + "{{ .Values.runCommand }}" + ]; + } + ]; + restartPolicy = "{{ .Values.restartPolicy }}"; }; }; }; }; - }; + passAsFile = [ + "values" + "chart" + "job" + ]; + } + '' + mkdir -p chart/templates + cp "$chartPath" chart/Chart.yaml + cp "$valuesPath" chart/values.yaml + cp "$jobPath" chart/templates/job.json - testScript = # python - let - manifestFormat = + helm package chart + mv ./*.tgz $out + ''; + # The common Helm chart that is used in this test + testChart = { + inherit package; + values = { + runCommand = "hello"; + image = { + repository = testImage.imageName; + tag = testImage.imageTag; + }; + }; + }; +in +{ + name = "${rancherPackage.name}-auto-deploy-helm"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + yq-go + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = { - k3s = "yaml"; - rke2 = "json"; + k3s = lib.remove "traefik" disabledComponents; + rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; } .${rancherDistro}; - in - '' - import json + images = + coreImages + # Provides the k3s Helm controller + ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images + ++ [ + testImage + ]; + autoDeployCharts = { + # regular test chart that should get installed + hello = testChart; + # disabled chart that should not get installed + disabled = testChart // { + enable = false; + }; + # chart with values set via YAML file + values-file = testChart // { + # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path + # https://github.com/NixOS/nix/issues/12407 + values = + /. + + builtins.unsafeDiscardStringContext ( + builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' + runCommand: "echo 'Hello, file!'" + image: + repository: test.local/test + tag: local + '' + ); + }; + # advanced chart that should get installed in the "test" namespace with a custom + # timeout and overridden values + advanced = testChart // { + # create the "test" namespace via extraDeploy for testing + extraDeploy = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "test"; + } + ]; + extraFieldDefinitions = { + spec = { + # overwrite chart values + valuesContent = '' + runCommand: "echo 'advanced hello'" + image: + repository: ${testImage.imageName} + tag: ${testImage.imageTag} + ''; + # overwrite the chart namespace + targetNamespace = "test"; + # set a custom timeout + timeout = "69s"; + }; + }; + }; + }; + }; + }; - machine.wait_for_unit("${serviceName}") - # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") - machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") - # check that the timeout is set correctly, select only the first item in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) - t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") - # wait for test jobs to complete - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) - machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) - # check output of test jobs - hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") - values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") - advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") - # strip the output to remove trailing whitespaces - t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") - t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") - t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") - # wait for bundled ingress deployment - ${ - { - k3s = '' - machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) - ''; - rke2 = '' - machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) - ''; - } - .${rancherDistro} + testScript = # python + let + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; } - ''; + .${rancherDistro}; + in + '' + import json - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + machine.wait_for_unit("${serviceName}") + # check existence/absence of chart manifest files + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") + # check that the timeout is set correctly, select only the first item in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) + t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") + # wait for test jobs to complete + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) + machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) + # check output of test jobs + hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") + values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") + advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") + # strip the output to remove trailing whitespaces + t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") + t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") + t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") + # wait for bundled ingress deployment + ${ + { + k3s = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + ''; + rke2 = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) + ''; + } + .${rancherDistro} + } + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix index 5d3abfff040f..a13d3d77c09b 100644 --- a/nixos/tests/rancher/auto-deploy.nix +++ b/nixos/tests/rancher/auto-deploy.nix @@ -1,136 +1,134 @@ # Tests whether container images are imported and auto deploying manifests work -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - pauseImageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = pauseImageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + pauseImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = pauseImageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + helloImage = pkgs.dockerTools.buildImage { + name = "test.local/hello"; + tag = "local"; + copyToRoot = pkgs.hello; + config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; + }; - manifestFormat = - { - k3s = "yaml"; - rke2 = "json"; - } - .${rancherDistro}; - in - { - name = "${rancherPackage.name}-auto-deploy"; + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-auto-deploy"; - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - cri-tools + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + cri-tools + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - images = coreImages ++ [ - pauseImage - helloImage - ]; - manifests = { - absent = { - enable = false; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "absent"; - }; + images = coreImages ++ [ + pauseImage + helloImage + ]; + manifests = { + absent = { + enable = false; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "absent"; }; + }; - present = { - target = "foo-namespace.${manifestFormat}"; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "foo"; - }; + present = { + target = "foo-namespace.${manifestFormat}"; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "foo"; }; + }; - hello.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "hello"; - spec = { - template.spec = { - containers = [ - { - name = "hello"; - image = "test.local/hello:local"; - } - ]; - restartPolicy = "OnFailure"; - }; + hello.content = { + apiVersion = "batch/v1"; + kind = "Job"; + metadata.name = "hello"; + spec = { + template.spec = { + containers = [ + { + name = "hello"; + image = "test.local/hello:local"; + } + ]; + restartPolicy = "OnFailure"; }; }; }; }; }; + }; - testScript = # python - '' - start_all() + testScript = # python + '' + start_all() - machine.wait_for_unit("${serviceName}") - # check existence of the manifest files - machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.wait_for_unit("${serviceName}") + # check existence of the manifest files + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") - # check if container images got imported - # for some reason, RKE2 also uses /run/k3s - machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") + # check if container images got imported + # for some reason, RKE2 also uses /run/k3s + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") - # check if resources of manifests got created - machine.wait_until_succeeds("kubectl get ns foo") - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") - machine.fail("kubectl get ns absent") - ''; + # check if resources of manifests got created + machine.wait_until_succeeds("kubectl get ns foo") + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") + machine.fail("kubectl get ns absent") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix index f7ab7af2e758..7d140b0f3088 100644 --- a/nixos/tests/rancher/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -1,61 +1,59 @@ # A test that containerdConfigTemplate settings get written to containerd/config.toml -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - nodeName = "test"; - in - { - name = "${rancherPackage.name}-containerd-config"; - nodes.machine = - { ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; +in +{ + name = "${rancherPackage.name}-containerd-config"; + nodes.machine = + { ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - virtualisation = vmResources; + virtualisation = vmResources; - services.${rancherDistro} = { - enable = true; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - inherit nodeName; - containerdConfigTemplate = '' - # Base ${rancherDistro} config - {{ template "base" . }} + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + containerdConfigTemplate = '' + # Base ${rancherDistro} config + {{ template "base" . }} - # MAGIC COMMENT - ''; - }; + # MAGIC COMMENT + ''; }; + }; - testScript = # python - '' - start_all() - machine.wait_for_unit("${serviceName}") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the config template file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") - t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") - # test whether the config file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") - t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") - ''; + testScript = # python + '' + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the config template file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") + t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") + # test whether the config file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") + t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix index 2ca5db896e32..a8d1e6f67fe6 100644 --- a/nixos/tests/rancher/default.nix +++ b/nixos/tests/rancher/default.nix @@ -1,101 +1,116 @@ { - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, + runTest, + pkgs, + lib, # service/package name to test rancherDistro, + ... }: let allPackages = lib.filterAttrs ( - n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success + name: package: + builtins.match "^${rancherDistro}(_[[:digit:]]+)+$" name != null + && (builtins.tryEval package).success ) pkgs; - mkTestArgs = rancherPackage: { - inherit - system - pkgs - rancherDistro - rancherPackage - ; + allTests = + let + mkTestArgs = rancherPackage: { + inherit rancherDistro rancherPackage; - # systemd service name - serviceName = - { - k3s = "k3s"; - rke2 = "rke2-server"; - } - .${rancherDistro}; - - # list passed to services.*.disable, - # for slightly reduced resource usage - disabledComponents = - { - k3s = [ - "coredns" - "local-storage" - "metrics-server" - "servicelb" - "traefik" - ]; - rke2 = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - } - .${rancherDistro}; - - # images that must be present for all tests - coreImages = - { - k3s = [ ]; - - rke2 = + # systemd service name + serviceName = { - aarch64-linux = [ - rancherPackage.images-core-linux-arm64-tar-zst - rancherPackage.images-canal-linux-arm64-tar-zst + k3s = "k3s"; + rke2 = "rke2-server"; + } + .${rancherDistro}; + + # list passed to services.*.disable, + # for slightly reduced resource usage + disabledComponents = + { + k3s = [ + "coredns" + "local-storage" + "metrics-server" + "servicelb" + "traefik" ]; - x86_64-linux = [ - rancherPackage.images-core-linux-amd64-tar-zst - rancherPackage.images-canal-linux-amd64-tar-zst + rke2 = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" ]; } - .${pkgs.stdenv.hostPlatform.system} - or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); - } - .${rancherDistro}; + .${rancherDistro}; - # virtualization.* attrs, since all distros - # need more resources than the default - vmResources = - { - k3s = { - memorySize = 1536; - diskSize = 4096; - }; - rke2 = { - cores = 4; - memorySize = 4096; - diskSize = 8092; - }; - } - .${rancherDistro}; - }; + # images that must be present for all tests + coreImages = + { + k3s = [ ]; - importTest = - path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages; + rke2 = + { + aarch64-linux = [ + rancherPackage.images-core-linux-arm64-tar-zst + rancherPackage.images-canal-linux-arm64-tar-zst + ]; + x86_64-linux = [ + rancherPackage.images-core-linux-amd64-tar-zst + rancherPackage.images-canal-linux-amd64-tar-zst + ]; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); + } + .${rancherDistro}; + + # virtualization.* attrs, since all distros + # need more resources than the default + vmResources = + { + k3s = { + memorySize = 1536; + diskSize = 4096; + }; + rke2 = { + cores = 4; + memorySize = 4096; + diskSize = 8092; + }; + } + .${rancherDistro}; + }; + + mkTests = + path: + lib.mapAttrs ( + name: package: + runTest { + imports = [ path ]; + _module.args = mkTestArgs package; + } + ) allPackages; + in + { + airgap-images = mkTests ./airgap-images.nix; + auto-deploy = mkTests ./auto-deploy.nix; + auto-deploy-charts = mkTests ./auto-deploy-charts.nix; + containerd-config = mkTests ./containerd-config.nix; + etcd = mkTests ./etcd.nix; + kubelet-config = mkTests ./kubelet-config.nix; + multi-node = mkTests ./multi-node.nix; + single-node = mkTests ./single-node.nix; + }; in -{ - airgap-images = importTest ./airgap-images.nix { }; - auto-deploy = importTest ./auto-deploy.nix { }; - auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; - containerd-config = importTest ./containerd-config.nix { }; - etcd = importTest ./etcd.nix { }; - kubelet-config = importTest ./kubelet-config.nix { }; - multi-node = importTest ./multi-node.nix { }; - single-node = importTest ./single-node.nix { }; + +allTests +// { + all = lib.concatMapAttrs ( + testType: lib.mapAttrs' (package: lib.nameValuePair "${testType}-${package}") + ) allTests; } diff --git a/nixos/tests/rancher/etcd.nix b/nixos/tests/rancher/etcd.nix index 0bc1941dd51a..d3db6530e0fb 100644 --- a/nixos/tests/rancher/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -1,131 +1,129 @@ # Tests K3s with Etcd backend -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: - { - name = "${rancherPackage.name}-etcd"; +{ + name = "${rancherPackage.name}-etcd"; - nodes = { + nodes = { - etcd = - { ... }: - { - services.etcd = { - enable = true; - openFirewall = true; - listenClientUrls = [ - "http://192.168.1.1:2379" - "http://127.0.0.1:2379" - ]; - listenPeerUrls = [ "http://192.168.1.1:2380" ]; - initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; - initialCluster = [ "etcd=http://192.168.1.1:2380" ]; - }; - networking = { - useDHCP = false; - defaultGateway = "192.168.1.1"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.1"; - prefixLength = 24; - } - ]; - }; - }; - - server = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq + etcd = + { ... }: + { + services.etcd = { + enable = true; + openFirewall = true; + listenClientUrls = [ + "http://192.168.1.1:2379" + "http://127.0.0.1:2379" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - nodeIP = "192.168.1.2"; - extraFlags = [ - "--datastore-endpoint=\"http://192.168.1.1:2379\"" - ]; - }; - - networking = { - firewall = { - allowedTCPPorts = [ - 2379 - 2380 - 6443 - ]; - allowedUDPPorts = [ 8472 ]; - }; - useDHCP = false; - defaultGateway = "192.168.1.2"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.2"; - prefixLength = 24; - } - ]; - }; + listenPeerUrls = [ "http://192.168.1.1:2380" ]; + initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; + initialCluster = [ "etcd=http://192.168.1.1:2380" ]; }; - }; + networking = { + useDHCP = false; + defaultGateway = "192.168.1.1"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } + ]; + }; + }; - testScript = # python - '' - with subtest("should start etcd"): - etcd.start() - etcd.wait_for_unit("etcd.service") + server = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - with subtest("should wait for etcdctl endpoint status to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint status") + virtualisation = vmResources; - with subtest("should wait for etcdctl endpoint health to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint health") + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + nodeIP = "192.168.1.2"; + extraFlags = [ + "--datastore-endpoint=\"http://192.168.1.1:2379\"" + ]; + }; - with subtest("should start ${rancherDistro}"): - server.start() - server.wait_for_unit("${serviceName}") + networking = { + firewall = { + allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + allowedUDPPorts = [ 8472 ]; + }; + useDHCP = false; + defaultGateway = "192.168.1.2"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } + ]; + }; + }; + }; - with subtest("should test if kubectl works"): - server.wait_until_succeeds("kubectl get node") + testScript = # python + '' + with subtest("should start etcd"): + etcd.start() + etcd.wait_for_unit("etcd.service") - with subtest("should wait for service account to show up; takes a sec"): - server.wait_until_succeeds("kubectl get serviceaccount default") + with subtest("should wait for etcdctl endpoint status to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint status") - with subtest("should create a sample secret object"): - server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + with subtest("should wait for etcdctl endpoint health to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint health") - with subtest("should check if secret is correct"): - server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + with subtest("should start ${rancherDistro}"): + server.start() + server.wait_for_unit("${serviceName}") - with subtest("should have a secret in database"): - etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + with subtest("should test if kubectl works"): + server.wait_until_succeeds("kubectl get node") - with subtest("should delete the secret"): - server.succeed("kubectl delete secret nixossecret") + with subtest("should wait for service account to show up; takes a sec"): + server.wait_until_succeeds("kubectl get serviceaccount default") - with subtest("should not have a secret in database"): - etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - ''; + with subtest("should create a sample secret object"): + server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") - meta.maintainers = - pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + with subtest("should check if secret is correct"): + server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + + with subtest("should have a secret in database"): + etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + + with subtest("should delete the secret"): + server.succeed("kubectl delete secret nixossecret") + + with subtest("should not have a secret in database"): + etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + ''; + + meta.maintainers = + pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix index fb921876a584..e0a3643901b9 100644 --- a/nixos/tests/rancher/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -1,77 +1,75 @@ # A test that sets extra kubelet configuration and enables graceful node shutdown -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - nodeName = "test"; - shutdownGracePeriod = "1m13s"; - shutdownGracePeriodCriticalPods = "13s"; - podsPerCore = 3; - memoryThrottlingFactor = 0.69; - containerLogMaxSize = "5Mi"; - in - { - name = "${rancherPackage.name}-kubelet-config"; - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; + shutdownGracePeriod = "1m13s"; + shutdownGracePeriodCriticalPods = "13s"; + podsPerCore = 3; + memoryThrottlingFactor = 0.69; + containerLogMaxSize = "5Mi"; +in +{ + name = "${rancherPackage.name}-kubelet-config"; + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - virtualisation = vmResources; + virtualisation = vmResources; - services.${rancherDistro} = { + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + gracefulNodeShutdown = { enable = true; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - inherit nodeName; - gracefulNodeShutdown = { - enable = true; - inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; - }; - extraKubeletConfig = { - inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; - }; + inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; + }; + extraKubeletConfig = { + inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; }; }; + }; - testScript = # python - '' - import json + testScript = # python + '' + import json - start_all() - machine.wait_for_unit("${serviceName}") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") - # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish - machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") - machine.wait_until_succeeds("nc -z 127.0.0.1 8001") - # get the kubeletconfig - kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the kubelet registered an inhibitor lock + machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") + # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish + machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") + machine.wait_until_succeeds("nc -z 127.0.0.1 8001") + # get the kubeletconfig + kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) - with subtest("Kubelet config values are set correctly"): - t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") - t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") - t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) - t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) - t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") - ''; + with subtest("Kubelet config values are set correctly"): + t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") + t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") + t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) + t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) + t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/multi-node.nix b/nixos/tests/rancher/multi-node.nix index acc077613d64..6e73d7996ecd 100644 --- a/nixos/tests/rancher/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -1,252 +1,250 @@ # A test that runs a multi-node rancher cluster and verifies pod networking works across nodes -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - bashInteractive - coreutils - socat - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - # A daemonset that responds 'server' on port 8000 - networkTestDaemonset = pkgs.writeText "test.yml" '' - apiVersion: apps/v1 - kind: DaemonSet - metadata: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + bashInteractive + coreutils + socat + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + # A daemonset that responds 'server' on port 8000 + networkTestDaemonset = pkgs.writeText "test.yml" '' + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: test + labels: name: test - labels: + spec: + selector: + matchLabels: name: test - spec: - selector: - matchLabels: + template: + metadata: + labels: name: test - template: - metadata: - labels: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - resources: - limits: - memory: 20Mi - command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] - ''; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + resources: + limits: + memory: 20Mi + command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] + ''; + tokenFile = pkgs.writeText "token" "p@s$w0rd"; - supervisorPort = + supervisorPort = + { + k3s = "6443"; + rke2 = "9345"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-multi-node"; + + nodes = { + server = { - k3s = "6443"; - rke2 = "9345"; - } - .${rancherDistro}; - in - { - name = "${rancherPackage.name}-multi-node"; + nodes, + pkgs, + config, + ... + }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - nodes = { - server = - { - nodes, - pkgs, - config, - ... - }: - { - environment.systemPackages = with pkgs; [ - kubectl - gzip - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + virtualisation = vmResources; - virtualisation = vmResources; - - services.${rancherDistro} = lib.mkMerge [ - { - inherit tokenFile; - enable = true; - role = "server"; - package = rancherPackage; - images = coreImages ++ [ pauseImage ]; - nodeIP = config.networking.primaryIPAddress; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - } - { - k3s = { - clusterInit = true; - extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition - }; - - # The interface selection logic of flannel & canal would normally use eth0, as - # the nixos testing driver sets a default route via dev eth0. However, in test - # setups we have to use eth1 for inter-node communication. - # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be - # configured with this manifest. - rke2.manifests.canal-config.content = { - apiVersion = "helm.cattle.io/v1"; - kind = "HelmChartConfig"; - metadata = { - name = "rke2-canal"; - namespace = "kube-system"; - }; - # spec.valuesContent needs to a string, either json or yaml - spec.valuesContent = builtins.toJSON { - flannel.iface = "eth1"; - }; - }; - } - .${rancherDistro} - ]; - - networking.firewall.enable = false; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 - ] - ++ lib.optionals (rancherDistro == "rke2") [ - 9099 - 9345 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - }; - - server2 = - { - nodes, - pkgs, - config, - ... - }: - { - virtualisation = vmResources; - - services.${rancherDistro} = { + services.${rancherDistro} = lib.mkMerge [ + { inherit tokenFile; enable = true; role = "server"; package = rancherPackage; images = coreImages ++ [ pauseImage ]; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; disable = disabledComponents; extraFlags = [ "--pause-image test.local/pause:local" - ] - ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; - }; + ]; + } + { + k3s = { + clusterInit = true; + extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition + }; - networking.firewall.enable = false; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 + # The interface selection logic of flannel & canal would normally use eth0, as + # the nixos testing driver sets a default route via dev eth0. However, in test + # setups we have to use eth1 for inter-node communication. + # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be + # configured with this manifest. + rke2.manifests.canal-config.content = { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChartConfig"; + metadata = { + name = "rke2-canal"; + namespace = "kube-system"; + }; + # spec.valuesContent needs to a string, either json or yaml + spec.valuesContent = builtins.toJSON { + flannel.iface = "eth1"; + }; + }; + } + .${rancherDistro} + ]; + + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + + server2 = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; + + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" ] - ++ lib.optionals (rancherDistro == "rke2") [ - 9099 - 9345 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - agent = - { - nodes, - pkgs, - config, - ... - }: - { - virtualisation = vmResources; + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; - services.${rancherDistro} = { - inherit tokenFile; - enable = true; - role = "agent"; - package = rancherPackage; - images = coreImages ++ [ pauseImage ]; - serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; - nodeIP = config.networking.primaryIPAddress; - extraFlags = [ - "--pause-image test.local/pause:local" - ] - ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; - }; + agent = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; - networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; - networking.firewall.allowedUDPPorts = [ 8472 ]; + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "agent"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + extraFlags = [ + "--pause-image test.local/pause:local" + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - }; - testScript = # python - '' - start_all() + networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + }; - servers = [server, server2] - for m in servers: - m.wait_for_unit("${serviceName}") + testScript = # python + '' + start_all() - # wait for the agent to show up - server.wait_until_succeeds("kubectl get node agent") + servers = [server, server2] + for m in servers: + m.wait_for_unit("${serviceName}") - ${lib.optionalString (rancherDistro == "k3s") '' - for m in machines: - m.succeed("k3s check-config") - ''} + # wait for the agent to show up + server.wait_until_succeeds("kubectl get node agent") - server.succeed("kubectl cluster-info") - # Also wait for our service account to show up; it takes a sec - server.wait_until_succeeds("kubectl get serviceaccount default") + ${lib.optionalString (rancherDistro == "k3s") '' + for m in machines: + m.succeed("k3s check-config") + ''} - # Now create a pod on each node via a daemonset and verify they can talk to each other. - server.succeed("kubectl apply -f ${networkTestDaemonset}") - server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') + server.succeed("kubectl cluster-info") + # Also wait for our service account to show up; it takes a sec + server.wait_until_succeeds("kubectl get serviceaccount default") - # Get pod IPs - pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() - pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] + # Now create a pod on each node via a daemonset and verify they can talk to each other. + server.succeed("kubectl apply -f ${networkTestDaemonset}") + server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') - # Verify each server can ping each pod ip - for pod_ip in pod_ips: - server.succeed(f"ping -c 1 {pod_ip}") - server2.succeed(f"ping -c 1 {pod_ip}") - agent.succeed(f"ping -c 1 {pod_ip}") - # Verify the pods can talk to each other - for pod in pods: - resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") - t.assertEqual(resp.strip(), "server") - ''; + # Get pod IPs + pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() + pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + # Verify each server can ping each pod ip + for pod_ip in pod_ips: + server.succeed(f"ping -c 1 {pod_ip}") + server2.succeed(f"ping -c 1 {pod_ip}") + agent.succeed(f"ping -c 1 {pod_ip}") + # Verify the pods can talk to each other + for pod in pods: + resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") + t.assertEqual(resp.strip(), "server") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/single-node.nix b/nixos/tests/rancher/single-node.nix index 57c1691afdee..b59d5486b97c 100644 --- a/nixos/tests/rancher/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -1,116 +1,114 @@ # A test that runs a single node rancher cluster and verifies a pod can run -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildLayeredImage { - name = "test.local/pause"; - tag = "local"; - contents = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - testPodYaml = pkgs.writeText "test.yaml" '' - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - command: ["sh", "-c", "sleep inf"] - ''; - in - { - name = "${rancherPackage.name}-single-node"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildLayeredImage { + name = "test.local/pause"; + tag = "local"; + contents = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + testPodYaml = pkgs.writeText "test.yaml" '' + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; +in +{ + name = "${rancherPackage.name}-single-node"; - nodes.machine = - { config, pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - gzip + nodes.machine = + { config, pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages ++ [ pauseImage ]; + extraFlags = [ + "--pause-image test.local/pause:local" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = coreImages ++ [ pauseImage ]; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - }; - - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access ${rancherDistro} by default"; - password = "*"; - }; - }; }; - testScript = # python - '' - start_all() + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access ${rancherDistro} by default"; + password = "*"; + }; + }; + }; - machine.wait_for_unit("${serviceName}") - machine.succeed("kubectl cluster-info") - machine.fail("sudo -u noprivs kubectl cluster-info") - ${lib.optionalString (rancherDistro == "k3s") '' - machine.succeed("k3s check-config") - ''} + testScript = # python + '' + start_all() - # Also wait for our service account to show up; it takes a sec - machine.wait_until_succeeds("kubectl get serviceaccount default") - machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") - machine.succeed("kubectl delete -f ${testPodYaml}") + machine.wait_for_unit("${serviceName}") + machine.succeed("kubectl cluster-info") + machine.fail("sudo -u noprivs kubectl cluster-info") + ${lib.optionalString (rancherDistro == "k3s") '' + machine.succeed("k3s check-config") + ''} - # regression test for #176445 - machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") + # Also wait for our service account to show up; it takes a sec + machine.wait_until_succeeds("kubectl get serviceaccount default") + machine.succeed("kubectl apply -f ${testPodYaml}") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") + machine.succeed("kubectl delete -f ${testPodYaml}") - with subtest("Run ${rancherDistro}-killall"): - # Call the killall script with a clean path to assert that - # all required commands are wrapped - output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") - t.assertNotIn("command not found", output, "killall script contains unknown command") + # regression test for #176445 + machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") - # Check that killall cleaned up properly - machine.fail("systemctl is-active ${serviceName}.service") - machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) - machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") - machine.fail("ip netns show | grep cni-") - ''; + with subtest("Run ${rancherDistro}-killall"): + # Call the killall script with a clean path to assert that + # all required commands are wrapped + output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") + t.assertNotIn("command not found", output, "killall script contains unknown command") - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + # Check that killall cleaned up properly + machine.fail("systemctl is-active ${serviceName}.service") + machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) + machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") + machine.fail("ip netns show | grep cni-") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/pkgs/applications/networking/cluster/k3s/builder.nix b/pkgs/applications/networking/cluster/k3s/builder.nix index ddb5a5221440..731e1e5955f8 100644 --- a/pkgs/applications/networking/cluster/k3s/builder.nix +++ b/pkgs/applications/networking/cluster/k3s/builder.nix @@ -470,14 +470,11 @@ buildGoModule (finalAttrs: { ; tests = let - mkTests = - version: - let - k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version); - in - lib.mapAttrs (name: value: nixosTests.k3s.${name}.${k3s_version}) nixosTests.k3s; + versionedPackage = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor k3sVersion); in - mkTests k3sVersion; + lib.mapAttrs (name: _: nixosTests.k3s.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.k3s + ); imagesList = throw "k3s.imagesList was removed"; airgapImages = throw "k3s.airgapImages was renamed to k3s.airgap-images"; airgapImagesAmd64 = throw "k3s.airgapImagesAmd64 was renamed to k3s.airgap-images-amd64-tar-zst"; diff --git a/pkgs/applications/networking/cluster/rke2/builder.nix b/pkgs/applications/networking/cluster/rke2/builder.nix index 99bfa58a1d82..63801f9d14f8 100644 --- a/pkgs/applications/networking/cluster/rke2/builder.nix +++ b/pkgs/applications/networking/cluster/rke2/builder.nix @@ -22,6 +22,7 @@ lib: makeWrapper, fetchzip, fetchurl, + versionCheckHook, # Runtime dependencies procps, @@ -42,7 +43,6 @@ lib: # Testing dependencies nixosTests, - testers, }: buildGoModule (finalAttrs: { pname = "rke2"; @@ -129,25 +129,19 @@ buildGoModule (finalAttrs: { go tool nm $out/bin/.rke2-wrapped | grep '_Cfunc__goboringcrypto_' > /dev/null runHook postInstallCheck ''; + nativeInstallCheckInputs = [ versionCheckHook ]; + versionCheckProgramArg = "--version"; passthru = { inherit updateScript; tests = let - moduleTests = - let - package_version = - "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); - in - lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2; + versionedPackage = + "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); in - { - version = testers.testVersion { - package = finalAttrs.finalPackage; - version = "v${finalAttrs.version}"; - }; - } - // moduleTests; + lib.mapAttrs (name: _: nixosTests.rke2.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.rke2 + ); } // (lib.mapAttrs (_: value: fetchurl value) imagesVersions);