From 49bd3e5ef62ca60cfaf19fd577c33c1adb6d3e6b Mon Sep 17 00:00:00 2001 From: azey Date: Thu, 11 Dec 2025 19:06:45 +0100 Subject: [PATCH] nixos/tests/{k3s,rke2}: migrate to runTest Also exposed a .all attr alongside existing tests, for easily running all tests for all package versions. --- nixos/tests/all-tests.nix | 16 +- nixos/tests/rancher/airgap-images.nix | 70 ++- nixos/tests/rancher/auto-deploy-charts.nix | 434 +++++++++-------- nixos/tests/rancher/auto-deploy.nix | 234 +++++----- nixos/tests/rancher/containerd-config.nix | 106 +++-- nixos/tests/rancher/default.nix | 183 ++++---- nixos/tests/rancher/etcd.nix | 224 +++++---- nixos/tests/rancher/kubelet-config.nix | 132 +++--- nixos/tests/rancher/multi-node.nix | 436 +++++++++--------- nixos/tests/rancher/single-node.nix | 208 +++++---- .../networking/cluster/k3s/builder.nix | 11 +- .../networking/cluster/rke2/builder.nix | 22 +- 12 files changed, 1039 insertions(+), 1037 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4859fd3b9df6..5f9bed4f1b7e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -800,7 +800,11 @@ in jitsi-meet = runTest ./jitsi-meet.nix; jool = import ./jool.nix { inherit pkgs runTest; }; jotta-cli = runTest ./jotta-cli.nix; - k3s = handleTest ./rancher { rancherDistro = "k3s"; }; + k3s = import ./rancher { + inherit pkgs runTest; + inherit (pkgs) lib; + rancherDistro = "k3s"; + }; kafka = handleTest ./kafka { }; kanboard = runTest ./web-apps/kanboard.nix; kanidm = runTest ./kanidm.nix; @@ -1346,7 +1350,15 @@ in restic-rest-server = runTest ./restic-rest-server.nix; retroarch = runTest ./retroarch.nix; ringboard = runTest ./ringboard.nix; - rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; }; + rke2 = import ./rancher { + inherit pkgs; + inherit (pkgs) lib; + runTest = runTestOn [ + "aarch64-linux" + "x86_64-linux" + ]; + rancherDistro = "rke2"; + }; rkvm = handleTest ./rkvm { }; rmfakecloud = runTest ./rmfakecloud.nix; robustirc-bridge = runTest ./robustirc-bridge.nix; diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix index cfc6e8da7f13..b9dabb5ddbf9 100644 --- a/nixos/tests/rancher/airgap-images.nix +++ b/nixos/tests/rancher/airgap-images.nix @@ -1,42 +1,40 @@ # A test that imports k3s airgapped images and verifies that all expected images are present -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - { - name = "${rancherPackage.name}-airgap-images"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +{ + name = "${rancherPackage.name}-airgap-images"; - nodes.machine = _: { - virtualisation = vmResources; + nodes.machine = _: { + virtualisation = vmResources; - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = - coreImages - ++ { - k3s = [ rancherPackage.airgap-images ]; - rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages - } - .${rancherDistro}; - }; + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = + coreImages + ++ { + k3s = [ rancherPackage.airgap-images ]; + rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages + } + .${rancherDistro}; }; + }; - testScript = '' - machine.wait_for_unit("${serviceName}") - machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") - ''; + testScript = '' + machine.wait_for_unit("${serviceName}") + machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix index e0a954b915f6..125f22790db8 100644 --- a/nixos/tests/rancher/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -1,232 +1,230 @@ # Tests whether container images are imported and auto deploying Helm charts, # including the bundled traefik or ingress-nginx, work -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - testImageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - busybox - hello - ]; - }; - testImage = pkgs.dockerTools.buildImage { - name = "test.local/test"; - tag = "local"; - # Slightly reduces the time needed to import image - compressor = "zstd"; - copyToRoot = testImageEnv; - }; - # pack the test helm chart as a .tgz archive - package = - pkgs.runCommand "${rancherDistro}-test-chart.tgz" - { - nativeBuildInputs = [ pkgs.kubernetes-helm ]; - chart = builtins.toJSON { - name = "${rancherDistro}-test-chart"; - version = "0.1.0"; - }; - values = builtins.toJSON { - restartPolicy = "Never"; - runCommand = ""; - image = { - repository = "foo"; - tag = "1.0.0"; - }; - }; - job = builtins.toJSON { - apiVersion = "batch/v1"; - kind = "Job"; - metadata = { - name = "{{ .Release.Name }}"; - namespace = "{{ .Release.Namespace }}"; - }; - spec = { - template = { - spec = { - containers = [ - { - name = "test"; - image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; - command = [ "sh" ]; - args = [ - "-c" - "{{ .Values.runCommand }}" - ]; - } - ]; - restartPolicy = "{{ .Values.restartPolicy }}"; - }; - }; - }; - }; - passAsFile = [ - "values" - "chart" - "job" - ]; - } - '' - mkdir -p chart/templates - cp "$chartPath" chart/Chart.yaml - cp "$valuesPath" chart/values.yaml - cp "$jobPath" chart/templates/job.json - - helm package chart - mv ./*.tgz $out - ''; - # The common Helm chart that is used in this test - testChart = { - inherit package; - values = { - runCommand = "hello"; - image = { - repository = testImage.imageName; - tag = testImage.imageTag; - }; - }; - }; - in - { - name = "${rancherPackage.name}-auto-deploy-helm"; - - nodes.machine = - { pkgs, ... }: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + testImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + busybox + hello + ]; + }; + testImage = pkgs.dockerTools.buildImage { + name = "test.local/test"; + tag = "local"; + # Slightly reduces the time needed to import image + compressor = "zstd"; + copyToRoot = testImageEnv; + }; + # pack the test helm chart as a .tgz archive + package = + pkgs.runCommand "${rancherDistro}-test-chart.tgz" { - environment.systemPackages = with pkgs; [ - kubectl - yq-go - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - package = rancherPackage; - disable = - { - k3s = lib.remove "traefik" disabledComponents; - rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; - } - .${rancherDistro}; - images = - coreImages - # Provides the k3s Helm controller - ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images - ++ [ - testImage - ]; - autoDeployCharts = { - # regular test chart that should get installed - hello = testChart; - # disabled chart that should not get installed - disabled = testChart // { - enable = false; - }; - # chart with values set via YAML file - values-file = testChart // { - # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path - # https://github.com/NixOS/nix/issues/12407 - values = - /. - + builtins.unsafeDiscardStringContext ( - builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' - runCommand: "echo 'Hello, file!'" - image: - repository: test.local/test - tag: local - '' - ); - }; - # advanced chart that should get installed in the "test" namespace with a custom - # timeout and overridden values - advanced = testChart // { - # create the "test" namespace via extraDeploy for testing - extraDeploy = [ - { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "test"; - } - ]; - extraFieldDefinitions = { - spec = { - # overwrite chart values - valuesContent = '' - runCommand: "echo 'advanced hello'" - image: - repository: ${testImage.imageName} - tag: ${testImage.imageTag} - ''; - # overwrite the chart namespace - targetNamespace = "test"; - # set a custom timeout - timeout = "69s"; - }; + nativeBuildInputs = [ pkgs.kubernetes-helm ]; + chart = builtins.toJSON { + name = "${rancherDistro}-test-chart"; + version = "0.1.0"; + }; + values = builtins.toJSON { + restartPolicy = "Never"; + runCommand = ""; + image = { + repository = "foo"; + tag = "1.0.0"; + }; + }; + job = builtins.toJSON { + apiVersion = "batch/v1"; + kind = "Job"; + metadata = { + name = "{{ .Release.Name }}"; + namespace = "{{ .Release.Namespace }}"; + }; + spec = { + template = { + spec = { + containers = [ + { + name = "test"; + image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; + command = [ "sh" ]; + args = [ + "-c" + "{{ .Values.runCommand }}" + ]; + } + ]; + restartPolicy = "{{ .Values.restartPolicy }}"; }; }; }; }; - }; + passAsFile = [ + "values" + "chart" + "job" + ]; + } + '' + mkdir -p chart/templates + cp "$chartPath" chart/Chart.yaml + cp "$valuesPath" chart/values.yaml + cp "$jobPath" chart/templates/job.json - testScript = # python - let - manifestFormat = + helm package chart + mv ./*.tgz $out + ''; + # The common Helm chart that is used in this test + testChart = { + inherit package; + values = { + runCommand = "hello"; + image = { + repository = testImage.imageName; + tag = testImage.imageTag; + }; + }; + }; +in +{ + name = "${rancherPackage.name}-auto-deploy-helm"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + yq-go + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = { - k3s = "yaml"; - rke2 = "json"; + k3s = lib.remove "traefik" disabledComponents; + rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; } .${rancherDistro}; - in - '' - import json + images = + coreImages + # Provides the k3s Helm controller + ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images + ++ [ + testImage + ]; + autoDeployCharts = { + # regular test chart that should get installed + hello = testChart; + # disabled chart that should not get installed + disabled = testChart // { + enable = false; + }; + # chart with values set via YAML file + values-file = testChart // { + # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path + # https://github.com/NixOS/nix/issues/12407 + values = + /. + + builtins.unsafeDiscardStringContext ( + builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' + runCommand: "echo 'Hello, file!'" + image: + repository: test.local/test + tag: local + '' + ); + }; + # advanced chart that should get installed in the "test" namespace with a custom + # timeout and overridden values + advanced = testChart // { + # create the "test" namespace via extraDeploy for testing + extraDeploy = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "test"; + } + ]; + extraFieldDefinitions = { + spec = { + # overwrite chart values + valuesContent = '' + runCommand: "echo 'advanced hello'" + image: + repository: ${testImage.imageName} + tag: ${testImage.imageTag} + ''; + # overwrite the chart namespace + targetNamespace = "test"; + # set a custom timeout + timeout = "69s"; + }; + }; + }; + }; + }; + }; - machine.wait_for_unit("${serviceName}") - # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") - machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") - # check that the timeout is set correctly, select only the first item in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) - t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") - # wait for test jobs to complete - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) - machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) - # check output of test jobs - hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") - values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") - advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") - # strip the output to remove trailing whitespaces - t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") - t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") - t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") - # wait for bundled ingress deployment - ${ - { - k3s = '' - machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) - ''; - rke2 = '' - machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) - ''; - } - .${rancherDistro} + testScript = # python + let + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; } - ''; + .${rancherDistro}; + in + '' + import json - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + machine.wait_for_unit("${serviceName}") + # check existence/absence of chart manifest files + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") + # check that the timeout is set correctly, select only the first item in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) + t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") + # wait for test jobs to complete + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) + machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) + # check output of test jobs + hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") + values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") + advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") + # strip the output to remove trailing whitespaces + t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") + t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") + t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") + # wait for bundled ingress deployment + ${ + { + k3s = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + ''; + rke2 = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) + ''; + } + .${rancherDistro} + } + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix index 5d3abfff040f..a13d3d77c09b 100644 --- a/nixos/tests/rancher/auto-deploy.nix +++ b/nixos/tests/rancher/auto-deploy.nix @@ -1,136 +1,134 @@ # Tests whether container images are imported and auto deploying manifests work -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - pauseImageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = pauseImageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + pauseImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = pauseImageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + helloImage = pkgs.dockerTools.buildImage { + name = "test.local/hello"; + tag = "local"; + copyToRoot = pkgs.hello; + config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; + }; - manifestFormat = - { - k3s = "yaml"; - rke2 = "json"; - } - .${rancherDistro}; - in - { - name = "${rancherPackage.name}-auto-deploy"; + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-auto-deploy"; - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - cri-tools + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + cri-tools + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - images = coreImages ++ [ - pauseImage - helloImage - ]; - manifests = { - absent = { - enable = false; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "absent"; - }; + images = coreImages ++ [ + pauseImage + helloImage + ]; + manifests = { + absent = { + enable = false; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "absent"; }; + }; - present = { - target = "foo-namespace.${manifestFormat}"; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "foo"; - }; + present = { + target = "foo-namespace.${manifestFormat}"; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "foo"; }; + }; - hello.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "hello"; - spec = { - template.spec = { - containers = [ - { - name = "hello"; - image = "test.local/hello:local"; - } - ]; - restartPolicy = "OnFailure"; - }; + hello.content = { + apiVersion = "batch/v1"; + kind = "Job"; + metadata.name = "hello"; + spec = { + template.spec = { + containers = [ + { + name = "hello"; + image = "test.local/hello:local"; + } + ]; + restartPolicy = "OnFailure"; }; }; }; }; }; + }; - testScript = # python - '' - start_all() + testScript = # python + '' + start_all() - machine.wait_for_unit("${serviceName}") - # check existence of the manifest files - machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.wait_for_unit("${serviceName}") + # check existence of the manifest files + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") - # check if container images got imported - # for some reason, RKE2 also uses /run/k3s - machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") + # check if container images got imported + # for some reason, RKE2 also uses /run/k3s + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") - # check if resources of manifests got created - machine.wait_until_succeeds("kubectl get ns foo") - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") - machine.fail("kubectl get ns absent") - ''; + # check if resources of manifests got created + machine.wait_until_succeeds("kubectl get ns foo") + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") + machine.fail("kubectl get ns absent") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix index f7ab7af2e758..7d140b0f3088 100644 --- a/nixos/tests/rancher/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -1,61 +1,59 @@ # A test that containerdConfigTemplate settings get written to containerd/config.toml -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - nodeName = "test"; - in - { - name = "${rancherPackage.name}-containerd-config"; - nodes.machine = - { ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; +in +{ + name = "${rancherPackage.name}-containerd-config"; + nodes.machine = + { ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - virtualisation = vmResources; + virtualisation = vmResources; - services.${rancherDistro} = { - enable = true; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - inherit nodeName; - containerdConfigTemplate = '' - # Base ${rancherDistro} config - {{ template "base" . }} + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + containerdConfigTemplate = '' + # Base ${rancherDistro} config + {{ template "base" . }} - # MAGIC COMMENT - ''; - }; + # MAGIC COMMENT + ''; }; + }; - testScript = # python - '' - start_all() - machine.wait_for_unit("${serviceName}") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the config template file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") - t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") - # test whether the config file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") - t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") - ''; + testScript = # python + '' + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the config template file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") + t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") + # test whether the config file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") + t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix index 2ca5db896e32..a8d1e6f67fe6 100644 --- a/nixos/tests/rancher/default.nix +++ b/nixos/tests/rancher/default.nix @@ -1,101 +1,116 @@ { - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, + runTest, + pkgs, + lib, # service/package name to test rancherDistro, + ... }: let allPackages = lib.filterAttrs ( - n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success + name: package: + builtins.match "^${rancherDistro}(_[[:digit:]]+)+$" name != null + && (builtins.tryEval package).success ) pkgs; - mkTestArgs = rancherPackage: { - inherit - system - pkgs - rancherDistro - rancherPackage - ; + allTests = + let + mkTestArgs = rancherPackage: { + inherit rancherDistro rancherPackage; - # systemd service name - serviceName = - { - k3s = "k3s"; - rke2 = "rke2-server"; - } - .${rancherDistro}; - - # list passed to services.*.disable, - # for slightly reduced resource usage - disabledComponents = - { - k3s = [ - "coredns" - "local-storage" - "metrics-server" - "servicelb" - "traefik" - ]; - rke2 = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - } - .${rancherDistro}; - - # images that must be present for all tests - coreImages = - { - k3s = [ ]; - - rke2 = + # systemd service name + serviceName = { - aarch64-linux = [ - rancherPackage.images-core-linux-arm64-tar-zst - rancherPackage.images-canal-linux-arm64-tar-zst + k3s = "k3s"; + rke2 = "rke2-server"; + } + .${rancherDistro}; + + # list passed to services.*.disable, + # for slightly reduced resource usage + disabledComponents = + { + k3s = [ + "coredns" + "local-storage" + "metrics-server" + "servicelb" + "traefik" ]; - x86_64-linux = [ - rancherPackage.images-core-linux-amd64-tar-zst - rancherPackage.images-canal-linux-amd64-tar-zst + rke2 = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" ]; } - .${pkgs.stdenv.hostPlatform.system} - or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); - } - .${rancherDistro}; + .${rancherDistro}; - # virtualization.* attrs, since all distros - # need more resources than the default - vmResources = - { - k3s = { - memorySize = 1536; - diskSize = 4096; - }; - rke2 = { - cores = 4; - memorySize = 4096; - diskSize = 8092; - }; - } - .${rancherDistro}; - }; + # images that must be present for all tests + coreImages = + { + k3s = [ ]; - importTest = - path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages; + rke2 = + { + aarch64-linux = [ + rancherPackage.images-core-linux-arm64-tar-zst + rancherPackage.images-canal-linux-arm64-tar-zst + ]; + x86_64-linux = [ + rancherPackage.images-core-linux-amd64-tar-zst + rancherPackage.images-canal-linux-amd64-tar-zst + ]; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); + } + .${rancherDistro}; + + # virtualization.* attrs, since all distros + # need more resources than the default + vmResources = + { + k3s = { + memorySize = 1536; + diskSize = 4096; + }; + rke2 = { + cores = 4; + memorySize = 4096; + diskSize = 8092; + }; + } + .${rancherDistro}; + }; + + mkTests = + path: + lib.mapAttrs ( + name: package: + runTest { + imports = [ path ]; + _module.args = mkTestArgs package; + } + ) allPackages; + in + { + airgap-images = mkTests ./airgap-images.nix; + auto-deploy = mkTests ./auto-deploy.nix; + auto-deploy-charts = mkTests ./auto-deploy-charts.nix; + containerd-config = mkTests ./containerd-config.nix; + etcd = mkTests ./etcd.nix; + kubelet-config = mkTests ./kubelet-config.nix; + multi-node = mkTests ./multi-node.nix; + single-node = mkTests ./single-node.nix; + }; in -{ - airgap-images = importTest ./airgap-images.nix { }; - auto-deploy = importTest ./auto-deploy.nix { }; - auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; - containerd-config = importTest ./containerd-config.nix { }; - etcd = importTest ./etcd.nix { }; - kubelet-config = importTest ./kubelet-config.nix { }; - multi-node = importTest ./multi-node.nix { }; - single-node = importTest ./single-node.nix { }; + +allTests +// { + all = lib.concatMapAttrs ( + testType: lib.mapAttrs' (package: lib.nameValuePair "${testType}-${package}") + ) allTests; } diff --git a/nixos/tests/rancher/etcd.nix b/nixos/tests/rancher/etcd.nix index 0bc1941dd51a..d3db6530e0fb 100644 --- a/nixos/tests/rancher/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -1,131 +1,129 @@ # Tests K3s with Etcd backend -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: - { - name = "${rancherPackage.name}-etcd"; +{ + name = "${rancherPackage.name}-etcd"; - nodes = { + nodes = { - etcd = - { ... }: - { - services.etcd = { - enable = true; - openFirewall = true; - listenClientUrls = [ - "http://192.168.1.1:2379" - "http://127.0.0.1:2379" - ]; - listenPeerUrls = [ "http://192.168.1.1:2380" ]; - initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; - initialCluster = [ "etcd=http://192.168.1.1:2380" ]; - }; - networking = { - useDHCP = false; - defaultGateway = "192.168.1.1"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.1"; - prefixLength = 24; - } - ]; - }; - }; - - server = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq + etcd = + { ... }: + { + services.etcd = { + enable = true; + openFirewall = true; + listenClientUrls = [ + "http://192.168.1.1:2379" + "http://127.0.0.1:2379" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - nodeIP = "192.168.1.2"; - extraFlags = [ - "--datastore-endpoint=\"http://192.168.1.1:2379\"" - ]; - }; - - networking = { - firewall = { - allowedTCPPorts = [ - 2379 - 2380 - 6443 - ]; - allowedUDPPorts = [ 8472 ]; - }; - useDHCP = false; - defaultGateway = "192.168.1.2"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.2"; - prefixLength = 24; - } - ]; - }; + listenPeerUrls = [ "http://192.168.1.1:2380" ]; + initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; + initialCluster = [ "etcd=http://192.168.1.1:2380" ]; }; - }; + networking = { + useDHCP = false; + defaultGateway = "192.168.1.1"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } + ]; + }; + }; - testScript = # python - '' - with subtest("should start etcd"): - etcd.start() - etcd.wait_for_unit("etcd.service") + server = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - with subtest("should wait for etcdctl endpoint status to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint status") + virtualisation = vmResources; - with subtest("should wait for etcdctl endpoint health to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint health") + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + nodeIP = "192.168.1.2"; + extraFlags = [ + "--datastore-endpoint=\"http://192.168.1.1:2379\"" + ]; + }; - with subtest("should start ${rancherDistro}"): - server.start() - server.wait_for_unit("${serviceName}") + networking = { + firewall = { + allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + allowedUDPPorts = [ 8472 ]; + }; + useDHCP = false; + defaultGateway = "192.168.1.2"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } + ]; + }; + }; + }; - with subtest("should test if kubectl works"): - server.wait_until_succeeds("kubectl get node") + testScript = # python + '' + with subtest("should start etcd"): + etcd.start() + etcd.wait_for_unit("etcd.service") - with subtest("should wait for service account to show up; takes a sec"): - server.wait_until_succeeds("kubectl get serviceaccount default") + with subtest("should wait for etcdctl endpoint status to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint status") - with subtest("should create a sample secret object"): - server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + with subtest("should wait for etcdctl endpoint health to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint health") - with subtest("should check if secret is correct"): - server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + with subtest("should start ${rancherDistro}"): + server.start() + server.wait_for_unit("${serviceName}") - with subtest("should have a secret in database"): - etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + with subtest("should test if kubectl works"): + server.wait_until_succeeds("kubectl get node") - with subtest("should delete the secret"): - server.succeed("kubectl delete secret nixossecret") + with subtest("should wait for service account to show up; takes a sec"): + server.wait_until_succeeds("kubectl get serviceaccount default") - with subtest("should not have a secret in database"): - etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - ''; + with subtest("should create a sample secret object"): + server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") - meta.maintainers = - pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + with subtest("should check if secret is correct"): + server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + + with subtest("should have a secret in database"): + etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + + with subtest("should delete the secret"): + server.succeed("kubectl delete secret nixossecret") + + with subtest("should not have a secret in database"): + etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + ''; + + meta.maintainers = + pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix index fb921876a584..e0a3643901b9 100644 --- a/nixos/tests/rancher/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -1,77 +1,75 @@ # A test that sets extra kubelet configuration and enables graceful node shutdown -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - nodeName = "test"; - shutdownGracePeriod = "1m13s"; - shutdownGracePeriodCriticalPods = "13s"; - podsPerCore = 3; - memoryThrottlingFactor = 0.69; - containerLogMaxSize = "5Mi"; - in - { - name = "${rancherPackage.name}-kubelet-config"; - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; + shutdownGracePeriod = "1m13s"; + shutdownGracePeriodCriticalPods = "13s"; + podsPerCore = 3; + memoryThrottlingFactor = 0.69; + containerLogMaxSize = "5Mi"; +in +{ + name = "${rancherPackage.name}-kubelet-config"; + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - virtualisation = vmResources; + virtualisation = vmResources; - services.${rancherDistro} = { + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + gracefulNodeShutdown = { enable = true; - package = rancherPackage; - disable = disabledComponents; - images = coreImages; - inherit nodeName; - gracefulNodeShutdown = { - enable = true; - inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; - }; - extraKubeletConfig = { - inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; - }; + inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; + }; + extraKubeletConfig = { + inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; }; }; + }; - testScript = # python - '' - import json + testScript = # python + '' + import json - start_all() - machine.wait_for_unit("${serviceName}") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") - # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish - machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") - machine.wait_until_succeeds("nc -z 127.0.0.1 8001") - # get the kubeletconfig - kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the kubelet registered an inhibitor lock + machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") + # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish + machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") + machine.wait_until_succeeds("nc -z 127.0.0.1 8001") + # get the kubeletconfig + kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) - with subtest("Kubelet config values are set correctly"): - t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") - t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") - t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) - t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) - t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") - ''; + with subtest("Kubelet config values are set correctly"): + t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") + t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") + t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) + t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) + t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") + ''; - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/multi-node.nix b/nixos/tests/rancher/multi-node.nix index acc077613d64..6e73d7996ecd 100644 --- a/nixos/tests/rancher/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -1,252 +1,250 @@ # A test that runs a multi-node rancher cluster and verifies pod networking works across nodes -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - bashInteractive - coreutils - socat - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - # A daemonset that responds 'server' on port 8000 - networkTestDaemonset = pkgs.writeText "test.yml" '' - apiVersion: apps/v1 - kind: DaemonSet - metadata: +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + bashInteractive + coreutils + socat + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + # A daemonset that responds 'server' on port 8000 + networkTestDaemonset = pkgs.writeText "test.yml" '' + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: test + labels: name: test - labels: + spec: + selector: + matchLabels: name: test - spec: - selector: - matchLabels: + template: + metadata: + labels: name: test - template: - metadata: - labels: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - resources: - limits: - memory: 20Mi - command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] - ''; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + resources: + limits: + memory: 20Mi + command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] + ''; + tokenFile = pkgs.writeText "token" "p@s$w0rd"; - supervisorPort = + supervisorPort = + { + k3s = "6443"; + rke2 = "9345"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-multi-node"; + + nodes = { + server = { - k3s = "6443"; - rke2 = "9345"; - } - .${rancherDistro}; - in - { - name = "${rancherPackage.name}-multi-node"; + nodes, + pkgs, + config, + ... + }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - nodes = { - server = - { - nodes, - pkgs, - config, - ... - }: - { - environment.systemPackages = with pkgs; [ - kubectl - gzip - jq - ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + virtualisation = vmResources; - virtualisation = vmResources; - - services.${rancherDistro} = lib.mkMerge [ - { - inherit tokenFile; - enable = true; - role = "server"; - package = rancherPackage; - images = coreImages ++ [ pauseImage ]; - nodeIP = config.networking.primaryIPAddress; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - } - { - k3s = { - clusterInit = true; - extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition - }; - - # The interface selection logic of flannel & canal would normally use eth0, as - # the nixos testing driver sets a default route via dev eth0. However, in test - # setups we have to use eth1 for inter-node communication. - # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be - # configured with this manifest. - rke2.manifests.canal-config.content = { - apiVersion = "helm.cattle.io/v1"; - kind = "HelmChartConfig"; - metadata = { - name = "rke2-canal"; - namespace = "kube-system"; - }; - # spec.valuesContent needs to a string, either json or yaml - spec.valuesContent = builtins.toJSON { - flannel.iface = "eth1"; - }; - }; - } - .${rancherDistro} - ]; - - networking.firewall.enable = false; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 - ] - ++ lib.optionals (rancherDistro == "rke2") [ - 9099 - 9345 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - }; - - server2 = - { - nodes, - pkgs, - config, - ... - }: - { - virtualisation = vmResources; - - services.${rancherDistro} = { + services.${rancherDistro} = lib.mkMerge [ + { inherit tokenFile; enable = true; role = "server"; package = rancherPackage; images = coreImages ++ [ pauseImage ]; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; disable = disabledComponents; extraFlags = [ "--pause-image test.local/pause:local" - ] - ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; - }; + ]; + } + { + k3s = { + clusterInit = true; + extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition + }; - networking.firewall.enable = false; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 + # The interface selection logic of flannel & canal would normally use eth0, as + # the nixos testing driver sets a default route via dev eth0. However, in test + # setups we have to use eth1 for inter-node communication. + # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be + # configured with this manifest. + rke2.manifests.canal-config.content = { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChartConfig"; + metadata = { + name = "rke2-canal"; + namespace = "kube-system"; + }; + # spec.valuesContent needs to a string, either json or yaml + spec.valuesContent = builtins.toJSON { + flannel.iface = "eth1"; + }; + }; + } + .${rancherDistro} + ]; + + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + + server2 = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; + + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" ] - ++ lib.optionals (rancherDistro == "rke2") [ - 9099 - 9345 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - agent = - { - nodes, - pkgs, - config, - ... - }: - { - virtualisation = vmResources; + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; - services.${rancherDistro} = { - inherit tokenFile; - enable = true; - role = "agent"; - package = rancherPackage; - images = coreImages ++ [ pauseImage ]; - serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; - nodeIP = config.networking.primaryIPAddress; - extraFlags = [ - "--pause-image test.local/pause:local" - ] - ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; - }; + agent = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; - networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; - networking.firewall.allowedUDPPorts = [ 8472 ]; + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "agent"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + extraFlags = [ + "--pause-image test.local/pause:local" + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - }; - testScript = # python - '' - start_all() + networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + }; - servers = [server, server2] - for m in servers: - m.wait_for_unit("${serviceName}") + testScript = # python + '' + start_all() - # wait for the agent to show up - server.wait_until_succeeds("kubectl get node agent") + servers = [server, server2] + for m in servers: + m.wait_for_unit("${serviceName}") - ${lib.optionalString (rancherDistro == "k3s") '' - for m in machines: - m.succeed("k3s check-config") - ''} + # wait for the agent to show up + server.wait_until_succeeds("kubectl get node agent") - server.succeed("kubectl cluster-info") - # Also wait for our service account to show up; it takes a sec - server.wait_until_succeeds("kubectl get serviceaccount default") + ${lib.optionalString (rancherDistro == "k3s") '' + for m in machines: + m.succeed("k3s check-config") + ''} - # Now create a pod on each node via a daemonset and verify they can talk to each other. - server.succeed("kubectl apply -f ${networkTestDaemonset}") - server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') + server.succeed("kubectl cluster-info") + # Also wait for our service account to show up; it takes a sec + server.wait_until_succeeds("kubectl get serviceaccount default") - # Get pod IPs - pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() - pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] + # Now create a pod on each node via a daemonset and verify they can talk to each other. + server.succeed("kubectl apply -f ${networkTestDaemonset}") + server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') - # Verify each server can ping each pod ip - for pod_ip in pod_ips: - server.succeed(f"ping -c 1 {pod_ip}") - server2.succeed(f"ping -c 1 {pod_ip}") - agent.succeed(f"ping -c 1 {pod_ip}") - # Verify the pods can talk to each other - for pod in pods: - resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") - t.assertEqual(resp.strip(), "server") - ''; + # Get pod IPs + pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() + pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + # Verify each server can ping each pod ip + for pod_ip in pod_ips: + server.succeed(f"ping -c 1 {pod_ip}") + server2.succeed(f"ping -c 1 {pod_ip}") + agent.succeed(f"ping -c 1 {pod_ip}") + # Verify the pods can talk to each other + for pod in pods: + resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") + t.assertEqual(resp.strip(), "server") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/single-node.nix b/nixos/tests/rancher/single-node.nix index 57c1691afdee..b59d5486b97c 100644 --- a/nixos/tests/rancher/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -1,116 +1,114 @@ # A test that runs a single node rancher cluster and verifies a pod can run -import ../make-test-python.nix ( - { - pkgs, - lib, - rancherDistro, - rancherPackage, - serviceName, - disabledComponents, - coreImages, - vmResources, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "${rancherDistro}-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildLayeredImage { - name = "test.local/pause"; - tag = "local"; - contents = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - testPodYaml = pkgs.writeText "test.yaml" '' - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - command: ["sh", "-c", "sleep inf"] - ''; - in - { - name = "${rancherPackage.name}-single-node"; +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildLayeredImage { + name = "test.local/pause"; + tag = "local"; + contents = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + testPodYaml = pkgs.writeText "test.yaml" '' + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; +in +{ + name = "${rancherPackage.name}-single-node"; - nodes.machine = - { config, pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - kubectl - gzip + nodes.machine = + { config, pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages ++ [ pauseImage ]; + extraFlags = [ + "--pause-image test.local/pause:local" ]; - environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - - virtualisation = vmResources; - - services.${rancherDistro} = { - enable = true; - role = "server"; - package = rancherPackage; - disable = disabledComponents; - images = coreImages ++ [ pauseImage ]; - extraFlags = [ - "--pause-image test.local/pause:local" - ]; - }; - - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access ${rancherDistro} by default"; - password = "*"; - }; - }; }; - testScript = # python - '' - start_all() + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access ${rancherDistro} by default"; + password = "*"; + }; + }; + }; - machine.wait_for_unit("${serviceName}") - machine.succeed("kubectl cluster-info") - machine.fail("sudo -u noprivs kubectl cluster-info") - ${lib.optionalString (rancherDistro == "k3s") '' - machine.succeed("k3s check-config") - ''} + testScript = # python + '' + start_all() - # Also wait for our service account to show up; it takes a sec - machine.wait_until_succeeds("kubectl get serviceaccount default") - machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") - machine.succeed("kubectl delete -f ${testPodYaml}") + machine.wait_for_unit("${serviceName}") + machine.succeed("kubectl cluster-info") + machine.fail("sudo -u noprivs kubectl cluster-info") + ${lib.optionalString (rancherDistro == "k3s") '' + machine.succeed("k3s check-config") + ''} - # regression test for #176445 - machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") + # Also wait for our service account to show up; it takes a sec + machine.wait_until_succeeds("kubectl get serviceaccount default") + machine.succeed("kubectl apply -f ${testPodYaml}") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") + machine.succeed("kubectl delete -f ${testPodYaml}") - with subtest("Run ${rancherDistro}-killall"): - # Call the killall script with a clean path to assert that - # all required commands are wrapped - output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") - t.assertNotIn("command not found", output, "killall script contains unknown command") + # regression test for #176445 + machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") - # Check that killall cleaned up properly - machine.fail("systemctl is-active ${serviceName}.service") - machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) - machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") - machine.fail("ip netns show | grep cni-") - ''; + with subtest("Run ${rancherDistro}-killall"): + # Call the killall script with a clean path to assert that + # all required commands are wrapped + output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") + t.assertNotIn("command not found", output, "killall script contains unknown command") - meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; - } -) + # Check that killall cleaned up properly + machine.fail("systemctl is-active ${serviceName}.service") + machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) + machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") + machine.fail("ip netns show | grep cni-") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/pkgs/applications/networking/cluster/k3s/builder.nix b/pkgs/applications/networking/cluster/k3s/builder.nix index ddb5a5221440..731e1e5955f8 100644 --- a/pkgs/applications/networking/cluster/k3s/builder.nix +++ b/pkgs/applications/networking/cluster/k3s/builder.nix @@ -470,14 +470,11 @@ buildGoModule (finalAttrs: { ; tests = let - mkTests = - version: - let - k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version); - in - lib.mapAttrs (name: value: nixosTests.k3s.${name}.${k3s_version}) nixosTests.k3s; + versionedPackage = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor k3sVersion); in - mkTests k3sVersion; + lib.mapAttrs (name: _: nixosTests.k3s.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.k3s + ); imagesList = throw "k3s.imagesList was removed"; airgapImages = throw "k3s.airgapImages was renamed to k3s.airgap-images"; airgapImagesAmd64 = throw "k3s.airgapImagesAmd64 was renamed to k3s.airgap-images-amd64-tar-zst"; diff --git a/pkgs/applications/networking/cluster/rke2/builder.nix b/pkgs/applications/networking/cluster/rke2/builder.nix index 99bfa58a1d82..63801f9d14f8 100644 --- a/pkgs/applications/networking/cluster/rke2/builder.nix +++ b/pkgs/applications/networking/cluster/rke2/builder.nix @@ -22,6 +22,7 @@ lib: makeWrapper, fetchzip, fetchurl, + versionCheckHook, # Runtime dependencies procps, @@ -42,7 +43,6 @@ lib: # Testing dependencies nixosTests, - testers, }: buildGoModule (finalAttrs: { pname = "rke2"; @@ -129,25 +129,19 @@ buildGoModule (finalAttrs: { go tool nm $out/bin/.rke2-wrapped | grep '_Cfunc__goboringcrypto_' > /dev/null runHook postInstallCheck ''; + nativeInstallCheckInputs = [ versionCheckHook ]; + versionCheckProgramArg = "--version"; passthru = { inherit updateScript; tests = let - moduleTests = - let - package_version = - "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); - in - lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2; + versionedPackage = + "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); in - { - version = testers.testVersion { - package = finalAttrs.finalPackage; - version = "v${finalAttrs.version}"; - }; - } - // moduleTests; + lib.mapAttrs (name: _: nixosTests.rke2.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.rke2 + ); } // (lib.mapAttrs (_: value: fetchurl value) imagesVersions);