From 2f5bd8e63ab497c7c2fb6e0e511c67a8dc4f073d Mon Sep 17 00:00:00 2001 From: azey Date: Wed, 10 Dec 2025 17:25:56 +0100 Subject: [PATCH] nixos/tests/k3s: generalize to rancher Similarly to 2ce16ee67eae, prepares for merge with nixos/tests/rke2. --- .github/labeler.yml | 2 +- nixos/tests/all-tests.nix | 2 +- nixos/tests/k3s/airgap-images.nix | 34 ----- nixos/tests/k3s/auto-deploy.nix | 124 ------------------ nixos/tests/k3s/default.nix | 34 ----- nixos/tests/rancher/airgap-images.nix | 38 ++++++ .../{k3s => rancher}/auto-deploy-charts.nix | 68 ++++++---- nixos/tests/rancher/auto-deploy.nix | 124 ++++++++++++++++++ .../{k3s => rancher}/containerd-config.nix | 30 ++--- nixos/tests/rancher/default.nix | 66 ++++++++++ nixos/tests/{k3s => rancher}/etcd.nix | 37 +++--- .../tests/{k3s => rancher}/kubelet-config.nix | 26 ++-- nixos/tests/{k3s => rancher}/multi-node.nix | 87 ++++++------ nixos/tests/{k3s => rancher}/single-node.nix | 50 +++---- .../cluster/k3s/docs/ONBOARDING_MAINTAINER.md | 2 +- 15 files changed, 390 insertions(+), 334 deletions(-) delete mode 100644 nixos/tests/k3s/airgap-images.nix delete mode 100644 nixos/tests/k3s/auto-deploy.nix delete mode 100644 nixos/tests/k3s/default.nix create mode 100644 nixos/tests/rancher/airgap-images.nix rename nixos/tests/{k3s => rancher}/auto-deploy-charts.nix (77%) create mode 100644 nixos/tests/rancher/auto-deploy.nix rename nixos/tests/{k3s => rancher}/containerd-config.nix (66%) create mode 100644 nixos/tests/rancher/default.nix rename nixos/tests/{k3s => rancher}/etcd.nix (77%) rename nixos/tests/{k3s => rancher}/kubelet-config.nix (83%) rename nixos/tests/{k3s => rancher}/multi-node.nix (69%) rename nixos/tests/{k3s => rancher}/single-node.nix (67%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 542e327e5026..bfbd131d21a5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -263,7 +263,7 @@ - any-glob-to-any-file: - nixos/modules/services/cluster/rancher/default.nix - nixos/modules/services/cluster/rancher/k3s.nix - - nixos/tests/k3s/**/* + - nixos/tests/rancher/**/* - pkgs/applications/networking/cluster/k3s/**/* "6.topic: kernel": diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index dea336cb1173..98bf23d40206 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -800,7 +800,7 @@ in jitsi-meet = runTest ./jitsi-meet.nix; jool = import ./jool.nix { inherit pkgs runTest; }; jotta-cli = runTest ./jotta-cli.nix; - k3s = handleTest ./k3s { }; + k3s = handleTest ./rancher { rancherDistro = "k3s"; }; kafka = handleTest ./kafka { }; kanboard = runTest ./web-apps/kanboard.nix; kanidm = runTest ./kanidm.nix; diff --git a/nixos/tests/k3s/airgap-images.nix b/nixos/tests/k3s/airgap-images.nix deleted file mode 100644 index 79fd2c77025b..000000000000 --- a/nixos/tests/k3s/airgap-images.nix +++ /dev/null @@ -1,34 +0,0 @@ -# A test that imports k3s airgapped images and verifies that all expected images are present -import ../make-test-python.nix ( - { lib, k3s, ... }: - { - name = "${k3s.name}-airgap-images"; - meta.maintainers = lib.teams.k3s.members; - - nodes.machine = _: { - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - role = "server"; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - ]; - images = [ k3s.airgap-images ]; - }; - }; - - testScript = '' - machine.wait_for_unit("k3s") - machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"") - ''; - } -) diff --git a/nixos/tests/k3s/auto-deploy.nix b/nixos/tests/k3s/auto-deploy.nix deleted file mode 100644 index 6539284b4e3e..000000000000 --- a/nixos/tests/k3s/auto-deploy.nix +++ /dev/null @@ -1,124 +0,0 @@ -# Tests whether container images are imported and auto deploying manifests work -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - pauseImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = pauseImageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - in - { - name = "${k3s.name}-auto-deploy"; - - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = [ k3s ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; - services.k3s.images = [ - pauseImage - helloImage - ]; - services.k3s.manifests = { - absent = { - enable = false; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "absent"; - }; - }; - - present = { - target = "foo-namespace.yaml"; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "foo"; - }; - }; - - hello.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "hello"; - spec = { - template.spec = { - containers = [ - { - name = "hello"; - image = "test.local/hello:local"; - } - ]; - restartPolicy = "OnFailure"; - }; - }; - }; - }; - }; - - testScript = # python - '' - start_all() - - machine.wait_for_unit("k3s") - # check existence of the manifest files - machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml") - - # check if container images got imported - machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") - - # check if resources of manifests got created - machine.wait_until_succeeds("kubectl get ns foo") - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") - machine.fail("kubectl get ns absent") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/default.nix b/nixos/tests/k3s/default.nix deleted file mode 100644 index 8d4203c1fb79..000000000000 --- a/nixos/tests/k3s/default.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allK3s = lib.filterAttrs ( - n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success - ) pkgs; -in -{ - airgap-images = lib.mapAttrs ( - _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; } - ) allK3s; - auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s; - auto-deploy-charts = lib.mapAttrs ( - _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; } - ) allK3s; - containerd-config = lib.mapAttrs ( - _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; } - ) allK3s; - etcd = lib.mapAttrs ( - _: k3s: - import ./etcd.nix { - inherit system pkgs k3s; - inherit (pkgs) etcd; - } - ) allK3s; - kubelet-config = lib.mapAttrs ( - _: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; } - ) allK3s; - multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s; - single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s; -} diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix new file mode 100644 index 000000000000..0cb5bfabdd19 --- /dev/null +++ b/nixos/tests/rancher/airgap-images.nix @@ -0,0 +1,38 @@ +# A test that imports k3s airgapped images and verifies that all expected images are present +import ../make-test-python.nix ( + { + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + ... + }: + { + name = "${rancherPackage.name}-airgap-images"; + meta.maintainers = lib.teams.k3s.members; + + nodes.machine = _: { + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = + { + k3s = [ rancherPackage.airgap-images ]; + } + .${rancherDistro}; + }; + }; + + testScript = '' + machine.wait_for_unit("${serviceName}") + machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") + ''; + } +) diff --git a/nixos/tests/k3s/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix similarity index 77% rename from nixos/tests/k3s/auto-deploy-charts.nix rename to nixos/tests/rancher/auto-deploy-charts.nix index b5d1a31f3c60..848a88c345b9 100644 --- a/nixos/tests/k3s/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -1,15 +1,18 @@ # Tests whether container images are imported and auto deploying Helm charts, -# including the bundled traefik, work +# including the bundled traefik or ingress-nginx, work import ../make-test-python.nix ( { - k3s, - lib, pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let testImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ busybox hello @@ -24,11 +27,11 @@ import ../make-test-python.nix ( }; # pack the test helm chart as a .tgz archive package = - pkgs.runCommand "k3s-test-chart.tgz" + pkgs.runCommand "${rancherDistro}-test-chart.tgz" { nativeBuildInputs = [ pkgs.kubernetes-helm ]; chart = builtins.toJSON { - name = "k3s-test-chart"; + name = "${rancherDistro}-test-chart"; version = "0.1.0"; }; values = builtins.toJSON { @@ -93,7 +96,7 @@ import ../make-test-python.nix ( }; in { - name = "${k3s.name}-auto-deploy-helm"; + name = "${rancherPackage.name}-auto-deploy-helm"; meta.maintainers = lib.teams.k3s.members; nodes.machine = { pkgs, ... }: @@ -104,19 +107,23 @@ import ../make-test-python.nix ( diskSize = 4096; }; environment.systemPackages = [ pkgs.yq-go ]; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - ]; + package = rancherPackage; + disable = + { + k3s = lib.remove "traefik" disabledComponents; + rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; + } + .${rancherDistro}; images = [ - # Provides the k3s Helm controller - k3s.airgap-images + { + # Provides the k3s Helm controller + k3s = rancherPackage.airgap-images; + rke2 = rancherPackage.images-core-linux-amd64-tar-zst; + } + .${rancherDistro} + testImage ]; autoDeployCharts = { @@ -133,7 +140,7 @@ import ../make-test-python.nix ( values = /. + builtins.unsafeDiscardStringContext ( - builtins.toFile "k3s-test-chart-values.yaml" '' + builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' runCommand: "echo 'Hello, file!'" image: repository: test.local/test @@ -176,14 +183,14 @@ import ../make-test-python.nix ( '' import json - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml") - machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml") # check that the timeout is set correctly, select only the first doc in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml")) + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")) t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") # wait for test jobs to complete machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) @@ -197,8 +204,15 @@ import ../make-test-python.nix ( t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") - # wait for bundled traefik deployment - machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + # wait for bundled ingress deployment + ${ + { + k3s = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + ''; + } + .${rancherDistro} + } ''; } ) diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix new file mode 100644 index 000000000000..74784f159cc9 --- /dev/null +++ b/nixos/tests/rancher/auto-deploy.nix @@ -0,0 +1,124 @@ +# Tests whether container images are imported and auto deploying manifests work +import ../make-test-python.nix ( + { + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + ... + }: + let + pauseImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = pauseImageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + helloImage = pkgs.dockerTools.buildImage { + name = "test.local/hello"; + tag = "local"; + copyToRoot = pkgs.hello; + config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; + }; + in + { + name = "${rancherPackage.name}-auto-deploy"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = [ rancherPackage ]; + + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + images = [ + pauseImage + helloImage + ]; + manifests = { + absent = { + enable = false; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "absent"; + }; + }; + + present = { + target = "foo-namespace.yaml"; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "foo"; + }; + }; + + hello.content = { + apiVersion = "batch/v1"; + kind = "Job"; + metadata.name = "hello"; + spec = { + template.spec = { + containers = [ + { + name = "hello"; + image = "test.local/hello:local"; + } + ]; + restartPolicy = "OnFailure"; + }; + }; + }; + }; + }; + }; + + testScript = # python + '' + start_all() + + machine.wait_for_unit("${serviceName}") + # check existence of the manifest files + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + + # check if container images got imported + machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") + + # check if resources of manifests got created + machine.wait_until_succeeds("kubectl get ns foo") + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") + machine.fail("kubectl get ns absent") + ''; + + meta.maintainers = lib.teams.k3s.members; + } +) diff --git a/nixos/tests/k3s/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix similarity index 66% rename from nixos/tests/k3s/containerd-config.nix rename to nixos/tests/rancher/containerd-config.nix index 0ebfd4dac347..a4b064edad21 100644 --- a/nixos/tests/k3s/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -3,14 +3,17 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let nodeName = "test"; in { - name = "${k3s.name}-containerd-config"; + name = "${rancherPackage.name}-containerd-config"; nodes.machine = { ... }: { @@ -19,20 +22,13 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; + package = rancherPackage; + disable = disabledComponents; + inherit nodeName; containerdConfigTemplate = '' - # Base K3s config + # Base ${rancherDistro} config {{ template "base" . }} # MAGIC COMMENT @@ -43,14 +39,14 @@ import ../make-test-python.nix ( testScript = # python '' start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the config template file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl") + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") # test whether the config file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml") + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") ''; diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix new file mode 100644 index 000000000000..89586b517bfd --- /dev/null +++ b/nixos/tests/rancher/default.nix @@ -0,0 +1,66 @@ +{ + system ? builtins.currentSystem, + pkgs ? import ../../.. { inherit system; }, + lib ? pkgs.lib, + # service/package name to test + rancherDistro, +}: +let + allPackages = lib.filterAttrs ( + n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success + ) pkgs; + + mkTestArgs = rancherPackage: { + inherit + system + pkgs + rancherDistro + rancherPackage + ; + + # systemd service name + serviceName = + { + k3s = "k3s"; + rke2 = "rke2-server"; + } + .${rancherDistro}; + + # list passed to services.*.disable, + # for slightly reduced resource usage + disabledComponents = + { + k3s = [ + "coredns" + "local-storage" + "metrics-server" + "servicelb" + "traefik" + ]; + rke2 = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" + ]; + } + .${rancherDistro}; + }; + + importTest = + path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages; +in +{ + airgap-images = importTest ./airgap-images.nix { }; + auto-deploy = importTest ./auto-deploy.nix { }; + auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; + containerd-config = importTest ./containerd-config.nix { }; + etcd = importTest ./etcd.nix { + inherit (pkgs) etcd; + }; + kubelet-config = importTest ./kubelet-config.nix { }; + multi-node = importTest ./multi-node.nix { }; + single-node = importTest ./single-node.nix { }; +} diff --git a/nixos/tests/k3s/etcd.nix b/nixos/tests/rancher/etcd.nix similarity index 77% rename from nixos/tests/k3s/etcd.nix rename to nixos/tests/rancher/etcd.nix index a44de1f6233f..2e40472d4911 100644 --- a/nixos/tests/k3s/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -3,13 +3,16 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, etcd, ... }: { - name = "${k3s.name}-etcd"; + name = "${rancherPackage.name}-etcd"; nodes = { @@ -39,7 +42,7 @@ import ../make-test-python.nix ( }; }; - k3s = + server = { pkgs, ... }: { environment.systemPackages = with pkgs; [ jq ]; @@ -47,18 +50,14 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; role = "server"; - package = k3s; + package = rancherPackage; + disable = disabledComponents; + nodeIP = "192.168.1.2"; extraFlags = [ "--datastore-endpoint=\"http://192.168.1.1:2379\"" - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-ip 192.168.1.2" ]; }; @@ -95,27 +94,27 @@ import ../make-test-python.nix ( with subtest("should wait for etcdctl endpoint health to succeed"): etcd.wait_until_succeeds("etcdctl endpoint health") - with subtest("should start k3s"): - k3s.start() - k3s.wait_for_unit("k3s") + with subtest("should start ${rancherDistro}"): + server.start() + server.wait_for_unit("${serviceName}") with subtest("should test if kubectl works"): - k3s.wait_until_succeeds("k3s kubectl get node") + server.wait_until_succeeds("kubectl get node") with subtest("should wait for service account to show up; takes a sec"): - k3s.wait_until_succeeds("k3s kubectl get serviceaccount default") + server.wait_until_succeeds("kubectl get serviceaccount default") with subtest("should create a sample secret object"): - k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") with subtest("should check if secret is correct"): - k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") with subtest("should have a secret in database"): etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") with subtest("should delete the secret"): - k3s.succeed("k3s kubectl delete secret nixossecret") + server.succeed("kubectl delete secret nixossecret") with subtest("should not have a secret in database"): etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") diff --git a/nixos/tests/k3s/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix similarity index 83% rename from nixos/tests/k3s/kubelet-config.nix rename to nixos/tests/rancher/kubelet-config.nix index f5aacd22a13f..322d40e69108 100644 --- a/nixos/tests/k3s/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -3,7 +3,10 @@ import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let @@ -15,7 +18,7 @@ import ../make-test-python.nix ( containerLogMaxSize = "5Mi"; in { - name = "${k3s.name}-kubelet-config"; + name = "${rancherPackage.name}-kubelet-config"; nodes.machine = { pkgs, ... }: { @@ -25,18 +28,11 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; + package = rancherPackage; + disable = disabledComponents; + inherit nodeName; gracefulNodeShutdown = { enable = true; inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; @@ -52,11 +48,11 @@ import ../make-test-python.nix ( import json start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"") + machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"") # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") machine.wait_until_succeeds("nc -z 127.0.0.1 8001") diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/rancher/multi-node.nix similarity index 69% rename from nixos/tests/k3s/multi-node.nix rename to nixos/tests/rancher/multi-node.nix index fdf825a3c103..14671a7ae59c 100644 --- a/nixos/tests/k3s/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -1,14 +1,17 @@ -# A test that runs a multi-node k3s cluster and verify pod networking works across nodes +# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ tini bashInteractive @@ -56,11 +59,16 @@ import ../make-test-python.nix ( tokenFile = pkgs.writeText "token" "p@s$w0rd"; in { - name = "${k3s.name}-multi-node"; + name = "${rancherPackage.name}-multi-node"; nodes = { server = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { environment.systemPackages = with pkgs; [ gzip @@ -70,21 +78,17 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "server"; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; clusterInit = true; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" "--pause-image test.local/pause:local" - "--node-ip ${nodes.server.networking.primaryIPAddress}" # The interface selection logic of flannel would normally use eth0, as the nixos # testing driver sets a default route via dev eth0. However, in test setups we # have to use eth1 for inter-node communication. @@ -100,7 +104,12 @@ import ../make-test-python.nix ( }; server2 = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { environment.systemPackages = with pkgs; [ gzip @@ -109,21 +118,17 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443"; clusterInit = false; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" "--pause-image test.local/pause:local" - "--node-ip ${nodes.server2.networking.primaryIPAddress}" "--flannel-iface eth1" ]; }; @@ -136,20 +141,25 @@ import ../make-test-python.nix ( }; agent = - { nodes, pkgs, ... }: + { + nodes, + pkgs, + config, + ... + }: { virtualisation.memorySize = 1024; virtualisation.diskSize = 2048; - services.k3s = { + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "agent"; - package = k3s; + package = rancherPackage; images = [ pauseImage ]; serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443"; + nodeIP = config.networking.primaryIPAddress; extraFlags = [ "--pause-image test.local/pause:local" - "--node-ip ${nodes.agent.networking.primaryIPAddress}" "--flannel-iface eth1" ]; }; @@ -164,25 +174,28 @@ import ../make-test-python.nix ( machines = [server, server2, agent] for m in machines: - m.wait_for_unit("k3s") + m.wait_for_unit("${serviceName}") # wait for the agent to show up - server.wait_until_succeeds("k3s kubectl get node agent") + server.wait_until_succeeds("kubectl get node agent") + server.succeed("kubectl get node >&2") - for m in machines: - m.succeed("k3s check-config") + ${lib.optionalString (rancherDistro == "k3s") '' + for m in machines: + m.succeed("k3s check-config") + ''} - server.succeed("k3s kubectl cluster-info") + server.succeed("kubectl cluster-info") # Also wait for our service account to show up; it takes a sec - server.wait_until_succeeds("k3s kubectl get serviceaccount default") + server.wait_until_succeeds("kubectl get serviceaccount default") # Now create a pod on each node via a daemonset and verify they can talk to each other. - server.succeed("k3s kubectl apply -f ${networkTestDaemonset}") - server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') + server.succeed("kubectl apply -f ${networkTestDaemonset}") + server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') # Get pod IPs - pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() - pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] + pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() + pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] # Verify each server can ping each pod ip for pod_ip in pod_ips: @@ -191,7 +204,7 @@ import ../make-test-python.nix ( agent.succeed(f"ping -c 1 {pod_ip}") # Verify the pods can talk to each other for pod in pods: - resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") + resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") t.assertEqual(resp.strip(), "server") ''; diff --git a/nixos/tests/k3s/single-node.nix b/nixos/tests/rancher/single-node.nix similarity index 67% rename from nixos/tests/k3s/single-node.nix rename to nixos/tests/rancher/single-node.nix index 0a6f688afddc..d894b951d798 100644 --- a/nixos/tests/k3s/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -1,14 +1,17 @@ -# A test that runs a single node k3s cluster and verify a pod can run +# A test that runs a single node rancher cluster and verifies a pod can run import ../make-test-python.nix ( { pkgs, lib, - k3s, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, ... }: let imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; + name = "${rancherDistro}-pause-image-env"; paths = with pkgs; [ tini (lib.hiPrio coreutils) @@ -40,13 +43,13 @@ import ../make-test-python.nix ( ''; in { - name = "${k3s.name}-single-node"; + name = "${rancherPackage.name}-single-node"; nodes.machine = { pkgs, ... }: { environment.systemPackages = with pkgs; [ - k3s + rancherPackage gzip ]; @@ -54,23 +57,20 @@ import ../make-test-python.nix ( virtualisation.memorySize = 1536; virtualisation.diskSize = 4096; - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + }; users.users = { noprivs = { isNormalUser = true; - description = "Can't access k3s by default"; + description = "Can't access ${rancherDistro} by default"; password = "*"; }; }; @@ -80,10 +80,12 @@ import ../make-test-python.nix ( '' start_all() - machine.wait_for_unit("k3s") + machine.wait_for_unit("${serviceName}") machine.succeed("kubectl cluster-info") machine.fail("sudo -u noprivs kubectl cluster-info") - machine.succeed("k3s check-config") + ${lib.optionalString (rancherDistro == "k3s") '' + machine.succeed("k3s check-config") + ''} machine.succeed( "${pauseImage} | ctr image import -" ) @@ -95,16 +97,16 @@ import ../make-test-python.nix ( machine.succeed("kubectl delete -f ${testPodYaml}") # regression test for #176445 - machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'") + machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") - with subtest("Run k3s-killall"): + with subtest("Run ${rancherDistro}-killall"): # Call the killall script with a clean path to assert that # all required commands are wrapped - output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr") + output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") t.assertNotIn("command not found", output, "killall script contains unknown command") # Check that killall cleaned up properly - machine.fail("systemctl is-active k3s.service") + machine.fail("systemctl is-active ${serviceName}.service") machine.fail("systemctl list-units | grep containerd") machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") machine.fail("ip netns show | grep cni-") diff --git a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md index 580c614bbee3..23eff54b0828 100644 --- a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md +++ b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md @@ -7,7 +7,7 @@ A K3s maintainer, maintains K3s's: - [documentation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/README.md) - [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s) - [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22) -- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s) +- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/rancher) - [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher) - [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating) - updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/)