diff --git a/.github/labeler.yml b/.github/labeler.yml index 542e327e5026..bfbd131d21a5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -263,7 +263,7 @@ - any-glob-to-any-file: - nixos/modules/services/cluster/rancher/default.nix - nixos/modules/services/cluster/rancher/k3s.nix - - nixos/tests/k3s/**/* + - nixos/tests/rancher/**/* - pkgs/applications/networking/cluster/k3s/**/* "6.topic: kernel": diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index cb7282078499..7717e9ee9253 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -802,7 +802,11 @@ in jitsi-meet = runTest ./jitsi-meet.nix; jool = import ./jool.nix { inherit pkgs runTest; }; jotta-cli = runTest ./jotta-cli.nix; - k3s = handleTest ./k3s { }; + k3s = import ./rancher { + inherit pkgs runTest; + inherit (pkgs) lib; + rancherDistro = "k3s"; + }; kafka = handleTest ./kafka { }; kaidan = runTest ./kaidan; kanboard = runTest ./web-apps/kanboard.nix; @@ -1343,7 +1347,15 @@ in restic-rest-server = runTest ./restic-rest-server.nix; retroarch = runTest ./retroarch.nix; ringboard = runTest ./ringboard.nix; - rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { }; + rke2 = import ./rancher { + inherit pkgs; + inherit (pkgs) lib; + runTest = runTestOn [ + "aarch64-linux" + "x86_64-linux" + ]; + rancherDistro = "rke2"; + }; rkvm = handleTest ./rkvm { }; rmfakecloud = runTest ./rmfakecloud.nix; robustirc-bridge = runTest ./robustirc-bridge.nix; diff --git a/nixos/tests/k3s/airgap-images.nix b/nixos/tests/k3s/airgap-images.nix deleted file mode 100644 index 79fd2c77025b..000000000000 --- a/nixos/tests/k3s/airgap-images.nix +++ /dev/null @@ -1,34 +0,0 @@ -# A test that imports k3s airgapped images and verifies that all expected images are present -import ../make-test-python.nix ( - { lib, k3s, ... }: - { - name = "${k3s.name}-airgap-images"; - meta.maintainers = lib.teams.k3s.members; - - nodes.machine = _: { - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - role = "server"; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - ]; - images = [ k3s.airgap-images ]; - }; - }; - - testScript = '' - machine.wait_for_unit("k3s") - machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"") - ''; - } -) diff --git a/nixos/tests/k3s/auto-deploy-charts.nix b/nixos/tests/k3s/auto-deploy-charts.nix deleted file mode 100644 index b5d1a31f3c60..000000000000 --- a/nixos/tests/k3s/auto-deploy-charts.nix +++ /dev/null @@ -1,204 +0,0 @@ -# Tests whether container images are imported and auto deploying Helm charts, -# including the bundled traefik, work -import ../make-test-python.nix ( - { - k3s, - lib, - pkgs, - ... - }: - let - testImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - busybox - hello - ]; - }; - testImage = pkgs.dockerTools.buildImage { - name = "test.local/test"; - tag = "local"; - # Slightly reduces the time needed to import image - compressor = "zstd"; - copyToRoot = testImageEnv; - }; - # pack the test helm chart as a .tgz archive - package = - pkgs.runCommand "k3s-test-chart.tgz" - { - nativeBuildInputs = [ pkgs.kubernetes-helm ]; - chart = builtins.toJSON { - name = "k3s-test-chart"; - version = "0.1.0"; - }; - values = builtins.toJSON { - restartPolicy = "Never"; - runCommand = ""; - image = { - repository = "foo"; - tag = "1.0.0"; - }; - }; - job = builtins.toJSON { - apiVersion = "batch/v1"; - kind = "Job"; - metadata = { - name = "{{ .Release.Name }}"; - namespace = "{{ .Release.Namespace }}"; - }; - spec = { - template = { - spec = { - containers = [ - { - name = "test"; - image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; - command = [ "sh" ]; - args = [ - "-c" - "{{ .Values.runCommand }}" - ]; - } - ]; - restartPolicy = "{{ .Values.restartPolicy }}"; - }; - }; - }; - }; - passAsFile = [ - "values" - "chart" - "job" - ]; - } - '' - mkdir -p chart/templates - cp "$chartPath" chart/Chart.yaml - cp "$valuesPath" chart/values.yaml - cp "$jobPath" chart/templates/job.json - - helm package chart - mv ./*.tgz $out - ''; - # The common Helm chart that is used in this test - testChart = { - inherit package; - values = { - runCommand = "hello"; - image = { - repository = testImage.imageName; - tag = testImage.imageTag; - }; - }; - }; - in - { - name = "${k3s.name}-auto-deploy-helm"; - meta.maintainers = lib.teams.k3s.members; - nodes.machine = - { pkgs, ... }: - { - # k3s uses enough resources the default vm fails. - virtualisation = { - memorySize = 1536; - diskSize = 4096; - }; - environment.systemPackages = [ pkgs.yq-go ]; - services.k3s = { - enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - ]; - images = [ - # Provides the k3s Helm controller - k3s.airgap-images - testImage - ]; - autoDeployCharts = { - # regular test chart that should get installed - hello = testChart; - # disabled chart that should not get installed - disabled = testChart // { - enable = false; - }; - # chart with values set via YAML file - values-file = testChart // { - # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path - # https://github.com/NixOS/nix/issues/12407 - values = - /. - + builtins.unsafeDiscardStringContext ( - builtins.toFile "k3s-test-chart-values.yaml" '' - runCommand: "echo 'Hello, file!'" - image: - repository: test.local/test - tag: local - '' - ); - }; - # advanced chart that should get installed in the "test" namespace with a custom - # timeout and overridden values - advanced = testChart // { - # create the "test" namespace via extraDeploy for testing - extraDeploy = [ - { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "test"; - } - ]; - extraFieldDefinitions = { - spec = { - # overwrite chart values - valuesContent = '' - runCommand: "echo 'advanced hello'" - image: - repository: ${testImage.imageName} - tag: ${testImage.imageTag} - ''; - # overwrite the chart namespace - targetNamespace = "test"; - # set a custom timeout - timeout = "69s"; - }; - }; - }; - }; - }; - }; - - testScript = # python - '' - import json - - machine.wait_for_unit("k3s") - # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml") - machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml") - machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml") - # check that the timeout is set correctly, select only the first doc in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml")) - t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") - # wait for test jobs to complete - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) - machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) - # check output of test jobs - hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") - values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") - advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") - # strip the output to remove trailing whitespaces - t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") - t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") - t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") - # wait for bundled traefik deployment - machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) - ''; - } -) diff --git a/nixos/tests/k3s/auto-deploy.nix b/nixos/tests/k3s/auto-deploy.nix deleted file mode 100644 index 6539284b4e3e..000000000000 --- a/nixos/tests/k3s/auto-deploy.nix +++ /dev/null @@ -1,124 +0,0 @@ -# Tests whether container images are imported and auto deploying manifests work -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - pauseImageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = pauseImageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - in - { - name = "${k3s.name}-auto-deploy"; - - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = [ k3s ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; - services.k3s.images = [ - pauseImage - helloImage - ]; - services.k3s.manifests = { - absent = { - enable = false; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "absent"; - }; - }; - - present = { - target = "foo-namespace.yaml"; - content = { - apiVersion = "v1"; - kind = "Namespace"; - metadata.name = "foo"; - }; - }; - - hello.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "hello"; - spec = { - template.spec = { - containers = [ - { - name = "hello"; - image = "test.local/hello:local"; - } - ]; - restartPolicy = "OnFailure"; - }; - }; - }; - }; - }; - - testScript = # python - '' - start_all() - - machine.wait_for_unit("k3s") - # check existence of the manifest files - machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml") - machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml") - - # check if container images got imported - machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") - - # check if resources of manifests got created - machine.wait_until_succeeds("kubectl get ns foo") - machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") - machine.fail("kubectl get ns absent") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/containerd-config.nix b/nixos/tests/k3s/containerd-config.nix deleted file mode 100644 index 0ebfd4dac347..000000000000 --- a/nixos/tests/k3s/containerd-config.nix +++ /dev/null @@ -1,59 +0,0 @@ -# A test that containerdConfigTemplate settings get written to containerd/config.toml -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - nodeName = "test"; - in - { - name = "${k3s.name}-containerd-config"; - nodes.machine = - { ... }: - { - environment.systemPackages = [ pkgs.jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; - containerdConfigTemplate = '' - # Base K3s config - {{ template "base" . }} - - # MAGIC COMMENT - ''; - }; - }; - - testScript = # python - '' - start_all() - machine.wait_for_unit("k3s") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the config template file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl") - t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") - # test whether the config file contains the magic comment - out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml") - t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/default.nix b/nixos/tests/k3s/default.nix deleted file mode 100644 index 8d4203c1fb79..000000000000 --- a/nixos/tests/k3s/default.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allK3s = lib.filterAttrs ( - n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success - ) pkgs; -in -{ - airgap-images = lib.mapAttrs ( - _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; } - ) allK3s; - auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s; - auto-deploy-charts = lib.mapAttrs ( - _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; } - ) allK3s; - containerd-config = lib.mapAttrs ( - _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; } - ) allK3s; - etcd = lib.mapAttrs ( - _: k3s: - import ./etcd.nix { - inherit system pkgs k3s; - inherit (pkgs) etcd; - } - ) allK3s; - kubelet-config = lib.mapAttrs ( - _: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; } - ) allK3s; - multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s; - single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s; -} diff --git a/nixos/tests/k3s/etcd.nix b/nixos/tests/k3s/etcd.nix deleted file mode 100644 index a44de1f6233f..000000000000 --- a/nixos/tests/k3s/etcd.nix +++ /dev/null @@ -1,126 +0,0 @@ -# Tests K3s with Etcd backend -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - etcd, - ... - }: - - { - name = "${k3s.name}-etcd"; - - nodes = { - - etcd = - { ... }: - { - services.etcd = { - enable = true; - openFirewall = true; - listenClientUrls = [ - "http://192.168.1.1:2379" - "http://127.0.0.1:2379" - ]; - listenPeerUrls = [ "http://192.168.1.1:2380" ]; - initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; - initialCluster = [ "etcd=http://192.168.1.1:2380" ]; - }; - networking = { - useDHCP = false; - defaultGateway = "192.168.1.1"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.1"; - prefixLength = 24; - } - ]; - }; - }; - - k3s = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - role = "server"; - package = k3s; - extraFlags = [ - "--datastore-endpoint=\"http://192.168.1.1:2379\"" - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-ip 192.168.1.2" - ]; - }; - - networking = { - firewall = { - allowedTCPPorts = [ - 2379 - 2380 - 6443 - ]; - allowedUDPPorts = [ 8472 ]; - }; - useDHCP = false; - defaultGateway = "192.168.1.2"; - interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ - { - address = "192.168.1.2"; - prefixLength = 24; - } - ]; - }; - }; - }; - - testScript = # python - '' - with subtest("should start etcd"): - etcd.start() - etcd.wait_for_unit("etcd.service") - - with subtest("should wait for etcdctl endpoint status to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint status") - - with subtest("should wait for etcdctl endpoint health to succeed"): - etcd.wait_until_succeeds("etcdctl endpoint health") - - with subtest("should start k3s"): - k3s.start() - k3s.wait_for_unit("k3s") - - with subtest("should test if kubectl works"): - k3s.wait_until_succeeds("k3s kubectl get node") - - with subtest("should wait for service account to show up; takes a sec"): - k3s.wait_until_succeeds("k3s kubectl get serviceaccount default") - - with subtest("should create a sample secret object"): - k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") - - with subtest("should check if secret is correct"): - k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") - - with subtest("should have a secret in database"): - etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - - with subtest("should delete the secret"): - k3s.succeed("k3s kubectl delete secret nixossecret") - - with subtest("should not have a secret in database"): - etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") - ''; - - meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/kubelet-config.nix b/nixos/tests/k3s/kubelet-config.nix deleted file mode 100644 index f5aacd22a13f..000000000000 --- a/nixos/tests/k3s/kubelet-config.nix +++ /dev/null @@ -1,76 +0,0 @@ -# A test that sets extra kubelet configuration and enables graceful node shutdown -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - nodeName = "test"; - shutdownGracePeriod = "1m13s"; - shutdownGracePeriodCriticalPods = "13s"; - podsPerCore = 3; - memoryThrottlingFactor = 0.69; - containerLogMaxSize = "5Mi"; - in - { - name = "${k3s.name}-kubelet-config"; - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - enable = true; - package = k3s; - # Slightly reduce resource usage - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--node-name ${nodeName}" - ]; - gracefulNodeShutdown = { - enable = true; - inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; - }; - extraKubeletConfig = { - inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; - }; - }; - }; - - testScript = # python - '' - import json - - start_all() - machine.wait_for_unit("k3s") - # wait until the node is ready - machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") - # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"") - # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish - machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") - machine.wait_until_succeeds("nc -z 127.0.0.1 8001") - # get the kubeletconfig - kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) - - with subtest("Kubelet config values are set correctly"): - t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") - t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") - t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) - t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) - t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/k3s/multi-node.nix deleted file mode 100644 index fdf825a3c103..000000000000 --- a/nixos/tests/k3s/multi-node.nix +++ /dev/null @@ -1,200 +0,0 @@ -# A test that runs a multi-node k3s cluster and verify pod networking works across nodes -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - tini - bashInteractive - coreutils - socat - ]; - }; - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - copyToRoot = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - # A daemonset that responds 'server' on port 8000 - networkTestDaemonset = pkgs.writeText "test.yml" '' - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: test - labels: - name: test - spec: - selector: - matchLabels: - name: test - template: - metadata: - labels: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - resources: - limits: - memory: 20Mi - command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] - ''; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; - in - { - name = "${k3s.name}-multi-node"; - - nodes = { - server = - { nodes, pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - gzip - jq - ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - inherit tokenFile; - enable = true; - role = "server"; - package = k3s; - images = [ pauseImage ]; - clusterInit = true; - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - "--node-ip ${nodes.server.networking.primaryIPAddress}" - # The interface selection logic of flannel would normally use eth0, as the nixos - # testing driver sets a default route via dev eth0. However, in test setups we - # have to use eth1 for inter-node communication. - "--flannel-iface eth1" - ]; - }; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - }; - - server2 = - { nodes, pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - gzip - jq - ]; - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s = { - inherit tokenFile; - enable = true; - package = k3s; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443"; - clusterInit = false; - extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - "--node-ip ${nodes.server2.networking.primaryIPAddress}" - "--flannel-iface eth1" - ]; - }; - networking.firewall.allowedTCPPorts = [ - 2379 - 2380 - 6443 - ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - }; - - agent = - { nodes, pkgs, ... }: - { - virtualisation.memorySize = 1024; - virtualisation.diskSize = 2048; - services.k3s = { - inherit tokenFile; - enable = true; - role = "agent"; - package = k3s; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443"; - extraFlags = [ - "--pause-image test.local/pause:local" - "--node-ip ${nodes.agent.networking.primaryIPAddress}" - "--flannel-iface eth1" - ]; - }; - networking.firewall.allowedTCPPorts = [ 6443 ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; - }; - }; - - testScript = # python - '' - start_all() - - machines = [server, server2, agent] - for m in machines: - m.wait_for_unit("k3s") - - # wait for the agent to show up - server.wait_until_succeeds("k3s kubectl get node agent") - - for m in machines: - m.succeed("k3s check-config") - - server.succeed("k3s kubectl cluster-info") - # Also wait for our service account to show up; it takes a sec - server.wait_until_succeeds("k3s kubectl get serviceaccount default") - - # Now create a pod on each node via a daemonset and verify they can talk to each other. - server.succeed("k3s kubectl apply -f ${networkTestDaemonset}") - server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') - - # Get pod IPs - pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() - pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] - - # Verify each server can ping each pod ip - for pod_ip in pod_ips: - server.succeed(f"ping -c 1 {pod_ip}") - server2.succeed(f"ping -c 1 {pod_ip}") - agent.succeed(f"ping -c 1 {pod_ip}") - # Verify the pods can talk to each other - for pod in pods: - resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") - t.assertEqual(resp.strip(), "server") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/k3s/single-node.nix b/nixos/tests/k3s/single-node.nix deleted file mode 100644 index 0a6f688afddc..000000000000 --- a/nixos/tests/k3s/single-node.nix +++ /dev/null @@ -1,115 +0,0 @@ -# A test that runs a single node k3s cluster and verify a pod can run -import ../make-test-python.nix ( - { - pkgs, - lib, - k3s, - ... - }: - let - imageEnv = pkgs.buildEnv { - name = "k3s-pause-image-env"; - paths = with pkgs; [ - tini - (lib.hiPrio coreutils) - busybox - ]; - }; - pauseImage = pkgs.dockerTools.streamLayeredImage { - name = "test.local/pause"; - tag = "local"; - contents = imageEnv; - config.Entrypoint = [ - "/bin/tini" - "--" - "/bin/sleep" - "inf" - ]; - }; - testPodYaml = pkgs.writeText "test.yml" '' - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - command: ["sh", "-c", "sleep inf"] - ''; - in - { - name = "${k3s.name}-single-node"; - - nodes.machine = - { pkgs, ... }: - { - environment.systemPackages = with pkgs; [ - k3s - gzip - ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; - - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = [ - "--disable coredns" - "--disable local-storage" - "--disable metrics-server" - "--disable servicelb" - "--disable traefik" - "--pause-image test.local/pause:local" - ]; - - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access k3s by default"; - password = "*"; - }; - }; - }; - - testScript = # python - '' - start_all() - - machine.wait_for_unit("k3s") - machine.succeed("kubectl cluster-info") - machine.fail("sudo -u noprivs kubectl cluster-info") - machine.succeed("k3s check-config") - machine.succeed( - "${pauseImage} | ctr image import -" - ) - - # Also wait for our service account to show up; it takes a sec - machine.wait_until_succeeds("kubectl get serviceaccount default") - machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test") - machine.succeed("kubectl delete -f ${testPodYaml}") - - # regression test for #176445 - machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'") - - with subtest("Run k3s-killall"): - # Call the killall script with a clean path to assert that - # all required commands are wrapped - output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr") - t.assertNotIn("command not found", output, "killall script contains unknown command") - - # Check that killall cleaned up properly - machine.fail("systemctl is-active k3s.service") - machine.fail("systemctl list-units | grep containerd") - machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") - machine.fail("ip netns show | grep cni-") - ''; - - meta.maintainers = lib.teams.k3s.members; - } -) diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix new file mode 100644 index 000000000000..b9dabb5ddbf9 --- /dev/null +++ b/nixos/tests/rancher/airgap-images.nix @@ -0,0 +1,40 @@ +# A test that imports k3s airgapped images and verifies that all expected images are present +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +{ + name = "${rancherPackage.name}-airgap-images"; + + nodes.machine = _: { + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = + coreImages + ++ { + k3s = [ rancherPackage.airgap-images ]; + rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages + } + .${rancherDistro}; + }; + }; + + testScript = '' + machine.wait_for_unit("${serviceName}") + machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix new file mode 100644 index 000000000000..125f22790db8 --- /dev/null +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -0,0 +1,230 @@ +# Tests whether container images are imported and auto deploying Helm charts, +# including the bundled traefik or ingress-nginx, work +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + testImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + busybox + hello + ]; + }; + testImage = pkgs.dockerTools.buildImage { + name = "test.local/test"; + tag = "local"; + # Slightly reduces the time needed to import image + compressor = "zstd"; + copyToRoot = testImageEnv; + }; + # pack the test helm chart as a .tgz archive + package = + pkgs.runCommand "${rancherDistro}-test-chart.tgz" + { + nativeBuildInputs = [ pkgs.kubernetes-helm ]; + chart = builtins.toJSON { + name = "${rancherDistro}-test-chart"; + version = "0.1.0"; + }; + values = builtins.toJSON { + restartPolicy = "Never"; + runCommand = ""; + image = { + repository = "foo"; + tag = "1.0.0"; + }; + }; + job = builtins.toJSON { + apiVersion = "batch/v1"; + kind = "Job"; + metadata = { + name = "{{ .Release.Name }}"; + namespace = "{{ .Release.Namespace }}"; + }; + spec = { + template = { + spec = { + containers = [ + { + name = "test"; + image = "{{ .Values.image.repository }}:{{ .Values.image.tag }}"; + command = [ "sh" ]; + args = [ + "-c" + "{{ .Values.runCommand }}" + ]; + } + ]; + restartPolicy = "{{ .Values.restartPolicy }}"; + }; + }; + }; + }; + passAsFile = [ + "values" + "chart" + "job" + ]; + } + '' + mkdir -p chart/templates + cp "$chartPath" chart/Chart.yaml + cp "$valuesPath" chart/values.yaml + cp "$jobPath" chart/templates/job.json + + helm package chart + mv ./*.tgz $out + ''; + # The common Helm chart that is used in this test + testChart = { + inherit package; + values = { + runCommand = "hello"; + image = { + repository = testImage.imageName; + tag = testImage.imageTag; + }; + }; + }; +in +{ + name = "${rancherPackage.name}-auto-deploy-helm"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + yq-go + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = + { + k3s = lib.remove "traefik" disabledComponents; + rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; + } + .${rancherDistro}; + images = + coreImages + # Provides the k3s Helm controller + ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images + ++ [ + testImage + ]; + autoDeployCharts = { + # regular test chart that should get installed + hello = testChart; + # disabled chart that should not get installed + disabled = testChart // { + enable = false; + }; + # chart with values set via YAML file + values-file = testChart // { + # Remove unsafeDiscardStringContext workaround when Nix can convert a string to a path + # https://github.com/NixOS/nix/issues/12407 + values = + /. + + builtins.unsafeDiscardStringContext ( + builtins.toFile "${rancherDistro}-test-chart-values.yaml" '' + runCommand: "echo 'Hello, file!'" + image: + repository: test.local/test + tag: local + '' + ); + }; + # advanced chart that should get installed in the "test" namespace with a custom + # timeout and overridden values + advanced = testChart // { + # create the "test" namespace via extraDeploy for testing + extraDeploy = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "test"; + } + ]; + extraFieldDefinitions = { + spec = { + # overwrite chart values + valuesContent = '' + runCommand: "echo 'advanced hello'" + image: + repository: ${testImage.imageName} + tag: ${testImage.imageTag} + ''; + # overwrite the chart namespace + targetNamespace = "test"; + # set a custom timeout + timeout = "69s"; + }; + }; + }; + }; + }; + }; + + testScript = # python + let + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; + in + '' + import json + + machine.wait_for_unit("${serviceName}") + # check existence/absence of chart manifest files + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") + # check that the timeout is set correctly, select only the first item in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) + t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") + # wait for test jobs to complete + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/values-file", timeout=180) + machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) + # check output of test jobs + hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") + values_file_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=values-file") + advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") + # strip the output to remove trailing whitespaces + t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job") + t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job") + t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job") + # wait for bundled ingress deployment + ${ + { + k3s = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) + ''; + rke2 = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) + ''; + } + .${rancherDistro} + } + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix new file mode 100644 index 000000000000..a13d3d77c09b --- /dev/null +++ b/nixos/tests/rancher/auto-deploy.nix @@ -0,0 +1,134 @@ +# Tests whether container images are imported and auto deploying manifests work +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + pauseImageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = pauseImageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + helloImage = pkgs.dockerTools.buildImage { + name = "test.local/hello"; + tag = "local"; + copyToRoot = pkgs.hello; + config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; + }; + + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-auto-deploy"; + + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + cri-tools + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + images = coreImages ++ [ + pauseImage + helloImage + ]; + manifests = { + absent = { + enable = false; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "absent"; + }; + }; + + present = { + target = "foo-namespace.${manifestFormat}"; + content = { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "foo"; + }; + }; + + hello.content = { + apiVersion = "batch/v1"; + kind = "Job"; + metadata.name = "hello"; + spec = { + template.spec = { + containers = [ + { + name = "hello"; + image = "test.local/hello:local"; + } + ]; + restartPolicy = "OnFailure"; + }; + }; + }; + }; + }; + }; + + testScript = # python + '' + start_all() + + machine.wait_for_unit("${serviceName}") + # check existence of the manifest files + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + + # check if container images got imported + # for some reason, RKE2 also uses /run/k3s + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") + + # check if resources of manifests got created + machine.wait_until_succeeds("kubectl get ns foo") + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello") + machine.fail("kubectl get ns absent") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix new file mode 100644 index 000000000000..7d140b0f3088 --- /dev/null +++ b/nixos/tests/rancher/containerd-config.nix @@ -0,0 +1,59 @@ +# A test that containerdConfigTemplate settings get written to containerd/config.toml +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; +in +{ + name = "${rancherPackage.name}-containerd-config"; + nodes.machine = + { ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + containerdConfigTemplate = '' + # Base ${rancherDistro} config + {{ template "base" . }} + + # MAGIC COMMENT + ''; + }; + }; + + testScript = # python + '' + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the config template file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl") + t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment") + # test whether the config file contains the magic comment + out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml") + t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix new file mode 100644 index 000000000000..a8d1e6f67fe6 --- /dev/null +++ b/nixos/tests/rancher/default.nix @@ -0,0 +1,116 @@ +{ + runTest, + pkgs, + lib, + # service/package name to test + rancherDistro, + ... +}: +let + allPackages = lib.filterAttrs ( + name: package: + builtins.match "^${rancherDistro}(_[[:digit:]]+)+$" name != null + && (builtins.tryEval package).success + ) pkgs; + + allTests = + let + mkTestArgs = rancherPackage: { + inherit rancherDistro rancherPackage; + + # systemd service name + serviceName = + { + k3s = "k3s"; + rke2 = "rke2-server"; + } + .${rancherDistro}; + + # list passed to services.*.disable, + # for slightly reduced resource usage + disabledComponents = + { + k3s = [ + "coredns" + "local-storage" + "metrics-server" + "servicelb" + "traefik" + ]; + rke2 = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" + ]; + } + .${rancherDistro}; + + # images that must be present for all tests + coreImages = + { + k3s = [ ]; + + rke2 = + { + aarch64-linux = [ + rancherPackage.images-core-linux-arm64-tar-zst + rancherPackage.images-canal-linux-arm64-tar-zst + ]; + x86_64-linux = [ + rancherPackage.images-core-linux-amd64-tar-zst + rancherPackage.images-canal-linux-amd64-tar-zst + ]; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); + } + .${rancherDistro}; + + # virtualization.* attrs, since all distros + # need more resources than the default + vmResources = + { + k3s = { + memorySize = 1536; + diskSize = 4096; + }; + rke2 = { + cores = 4; + memorySize = 4096; + diskSize = 8092; + }; + } + .${rancherDistro}; + }; + + mkTests = + path: + lib.mapAttrs ( + name: package: + runTest { + imports = [ path ]; + _module.args = mkTestArgs package; + } + ) allPackages; + in + { + airgap-images = mkTests ./airgap-images.nix; + auto-deploy = mkTests ./auto-deploy.nix; + auto-deploy-charts = mkTests ./auto-deploy-charts.nix; + containerd-config = mkTests ./containerd-config.nix; + etcd = mkTests ./etcd.nix; + kubelet-config = mkTests ./kubelet-config.nix; + multi-node = mkTests ./multi-node.nix; + single-node = mkTests ./single-node.nix; + }; +in + +allTests +// { + all = lib.concatMapAttrs ( + testType: lib.mapAttrs' (package: lib.nameValuePair "${testType}-${package}") + ) allTests; +} diff --git a/nixos/tests/rancher/etcd.nix b/nixos/tests/rancher/etcd.nix new file mode 100644 index 000000000000..d3db6530e0fb --- /dev/null +++ b/nixos/tests/rancher/etcd.nix @@ -0,0 +1,129 @@ +# Tests K3s with Etcd backend +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: + +{ + name = "${rancherPackage.name}-etcd"; + + nodes = { + + etcd = + { ... }: + { + services.etcd = { + enable = true; + openFirewall = true; + listenClientUrls = [ + "http://192.168.1.1:2379" + "http://127.0.0.1:2379" + ]; + listenPeerUrls = [ "http://192.168.1.1:2380" ]; + initialAdvertisePeerUrls = [ "http://192.168.1.1:2380" ]; + initialCluster = [ "etcd=http://192.168.1.1:2380" ]; + }; + networking = { + useDHCP = false; + defaultGateway = "192.168.1.1"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.1"; + prefixLength = 24; + } + ]; + }; + }; + + server = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + nodeIP = "192.168.1.2"; + extraFlags = [ + "--datastore-endpoint=\"http://192.168.1.1:2379\"" + ]; + }; + + networking = { + firewall = { + allowedTCPPorts = [ + 2379 + 2380 + 6443 + ]; + allowedUDPPorts = [ 8472 ]; + }; + useDHCP = false; + defaultGateway = "192.168.1.2"; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { + address = "192.168.1.2"; + prefixLength = 24; + } + ]; + }; + }; + }; + + testScript = # python + '' + with subtest("should start etcd"): + etcd.start() + etcd.wait_for_unit("etcd.service") + + with subtest("should wait for etcdctl endpoint status to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint status") + + with subtest("should wait for etcdctl endpoint health to succeed"): + etcd.wait_until_succeeds("etcdctl endpoint health") + + with subtest("should start ${rancherDistro}"): + server.start() + server.wait_for_unit("${serviceName}") + + with subtest("should test if kubectl works"): + server.wait_until_succeeds("kubectl get node") + + with subtest("should wait for service account to show up; takes a sec"): + server.wait_until_succeeds("kubectl get serviceaccount default") + + with subtest("should create a sample secret object"): + server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra") + + with subtest("should check if secret is correct"): + server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]") + + with subtest("should have a secret in database"): + etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + + with subtest("should delete the secret"): + server.succeed("kubectl delete secret nixossecret") + + with subtest("should not have a secret in database"): + etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") + ''; + + meta.maintainers = + pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix new file mode 100644 index 000000000000..e0a3643901b9 --- /dev/null +++ b/nixos/tests/rancher/kubelet-config.nix @@ -0,0 +1,75 @@ +# A test that sets extra kubelet configuration and enables graceful node shutdown +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + nodeName = "test"; + shutdownGracePeriod = "1m13s"; + shutdownGracePeriodCriticalPods = "13s"; + podsPerCore = 3; + memoryThrottlingFactor = 0.69; + containerLogMaxSize = "5Mi"; +in +{ + name = "${rancherPackage.name}-kubelet-config"; + nodes.machine = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + package = rancherPackage; + disable = disabledComponents; + images = coreImages; + inherit nodeName; + gracefulNodeShutdown = { + enable = true; + inherit shutdownGracePeriod shutdownGracePeriodCriticalPods; + }; + extraKubeletConfig = { + inherit podsPerCore memoryThrottlingFactor containerLogMaxSize; + }; + }; + }; + + testScript = # python + '' + import json + + start_all() + machine.wait_for_unit("${serviceName}") + # wait until the node is ready + machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") + # test whether the kubelet registered an inhibitor lock + machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") + # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish + machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") + machine.wait_until_succeeds("nc -z 127.0.0.1 8001") + # get the kubeletconfig + kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'")) + + with subtest("Kubelet config values are set correctly"): + t.assertEqual(kubelet_config["shutdownGracePeriod"], "${shutdownGracePeriod}") + t.assertEqual(kubelet_config["shutdownGracePeriodCriticalPods"], "${shutdownGracePeriodCriticalPods}") + t.assertEqual(kubelet_config["podsPerCore"], ${toString podsPerCore}) + t.assertEqual(kubelet_config["memoryThrottlingFactor"], ${toString memoryThrottlingFactor}) + t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/multi-node.nix b/nixos/tests/rancher/multi-node.nix new file mode 100644 index 000000000000..6e73d7996ecd --- /dev/null +++ b/nixos/tests/rancher/multi-node.nix @@ -0,0 +1,250 @@ +# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + bashInteractive + coreutils + socat + ]; + }; + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + copyToRoot = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + # A daemonset that responds 'server' on port 8000 + networkTestDaemonset = pkgs.writeText "test.yml" '' + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: test + labels: + name: test + spec: + selector: + matchLabels: + name: test + template: + metadata: + labels: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + resources: + limits: + memory: 20Mi + command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] + ''; + tokenFile = pkgs.writeText "token" "p@s$w0rd"; + + supervisorPort = + { + k3s = "6443"; + rke2 = "9345"; + } + .${rancherDistro}; +in +{ + name = "${rancherPackage.name}-multi-node"; + + nodes = { + server = + { + nodes, + pkgs, + config, + ... + }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = lib.mkMerge [ + { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + } + { + k3s = { + clusterInit = true; + extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition + }; + + # The interface selection logic of flannel & canal would normally use eth0, as + # the nixos testing driver sets a default route via dev eth0. However, in test + # setups we have to use eth1 for inter-node communication. + # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be + # configured with this manifest. + rke2.manifests.canal-config.content = { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChartConfig"; + metadata = { + name = "rke2-canal"; + namespace = "kube-system"; + }; + # spec.valuesContent needs to a string, either json or yaml + spec.valuesContent = builtins.toJSON { + flannel.iface = "eth1"; + }; + }; + } + .${rancherDistro} + ]; + + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + + server2 = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; + + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; + }; + + networking.firewall.enable = false; + networking.firewall.allowedTCPPorts = [ + 2379 + 2380 + 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 + ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + + agent = + { + nodes, + pkgs, + config, + ... + }: + { + virtualisation = vmResources; + + services.${rancherDistro} = { + inherit tokenFile; + enable = true; + role = "agent"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; + nodeIP = config.networking.primaryIPAddress; + extraFlags = [ + "--pause-image test.local/pause:local" + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; + }; + + networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; + networking.firewall.allowedUDPPorts = [ 8472 ]; + }; + }; + + testScript = # python + '' + start_all() + + servers = [server, server2] + for m in servers: + m.wait_for_unit("${serviceName}") + + # wait for the agent to show up + server.wait_until_succeeds("kubectl get node agent") + + ${lib.optionalString (rancherDistro == "k3s") '' + for m in machines: + m.succeed("k3s check-config") + ''} + + server.succeed("kubectl cluster-info") + # Also wait for our service account to show up; it takes a sec + server.wait_until_succeeds("kubectl get serviceaccount default") + + # Now create a pod on each node via a daemonset and verify they can talk to each other. + server.succeed("kubectl apply -f ${networkTestDaemonset}") + server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]') + + # Get pod IPs + pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines() + pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods] + + # Verify each server can ping each pod ip + for pod_ip in pod_ips: + server.succeed(f"ping -c 1 {pod_ip}") + server2.succeed(f"ping -c 1 {pod_ip}") + agent.succeed(f"ping -c 1 {pod_ip}") + # Verify the pods can talk to each other + for pod in pods: + resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -") + t.assertEqual(resp.strip(), "server") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rancher/single-node.nix b/nixos/tests/rancher/single-node.nix new file mode 100644 index 000000000000..b59d5486b97c --- /dev/null +++ b/nixos/tests/rancher/single-node.nix @@ -0,0 +1,114 @@ +# A test that runs a single node rancher cluster and verifies a pod can run +{ + pkgs, + lib, + rancherDistro, + rancherPackage, + serviceName, + disabledComponents, + coreImages, + vmResources, + ... +}: +let + imageEnv = pkgs.buildEnv { + name = "${rancherDistro}-pause-image-env"; + paths = with pkgs; [ + tini + (lib.hiPrio coreutils) + busybox + ]; + }; + pauseImage = pkgs.dockerTools.buildLayeredImage { + name = "test.local/pause"; + tag = "local"; + contents = imageEnv; + config.Entrypoint = [ + "/bin/tini" + "--" + "/bin/sleep" + "inf" + ]; + }; + testPodYaml = pkgs.writeText "test.yaml" '' + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; +in +{ + name = "${rancherPackage.name}-single-node"; + + nodes.machine = + { config, pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + kubectl + gzip + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + + services.${rancherDistro} = { + enable = true; + role = "server"; + package = rancherPackage; + disable = disabledComponents; + images = coreImages ++ [ pauseImage ]; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + }; + + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access ${rancherDistro} by default"; + password = "*"; + }; + }; + }; + + testScript = # python + '' + start_all() + + machine.wait_for_unit("${serviceName}") + machine.succeed("kubectl cluster-info") + machine.fail("sudo -u noprivs kubectl cluster-info") + ${lib.optionalString (rancherDistro == "k3s") '' + machine.succeed("k3s check-config") + ''} + + # Also wait for our service account to show up; it takes a sec + machine.wait_until_succeeds("kubectl get serviceaccount default") + machine.succeed("kubectl apply -f ${testPodYaml}") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") + machine.succeed("kubectl delete -f ${testPodYaml}") + + # regression test for #176445 + machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'") + + with subtest("Run ${rancherDistro}-killall"): + # Call the killall script with a clean path to assert that + # all required commands are wrapped + output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr") + t.assertNotIn("command not found", output, "killall script contains unknown command") + + # Check that killall cleaned up properly + machine.fail("systemctl is-active ${serviceName}.service") + machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) + machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") + machine.fail("ip netns show | grep cni-") + ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; +} diff --git a/nixos/tests/rke2/default.nix b/nixos/tests/rke2/default.nix deleted file mode 100644 index 90415a262acf..000000000000 --- a/nixos/tests/rke2/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs; -in -{ - # Run a single node rke2 cluster and verify a pod can run - singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2; - # Run a multi-node rke2 cluster and verify pod networking works across nodes - multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2; -} diff --git a/nixos/tests/rke2/multi-node.nix b/nixos/tests/rke2/multi-node.nix deleted file mode 100644 index ee394c07f828..000000000000 --- a/nixos/tests/rke2/multi-node.nix +++ /dev/null @@ -1,207 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.buildEnv { - name = "rke2-hello-image-env"; - paths = with pkgs; [ - coreutils - socat - ]; - }; - }; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; - agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd"; - # Let flannel use eth1 to enable inter-node communication in tests - canalConfig = { - apiVersion = "helm.cattle.io/v1"; - kind = "HelmChartConfig"; - metadata = { - name = "rke2-canal"; - namespace = "kube-system"; - }; - # spec.valuesContent needs to a string, either json or yaml - spec.valuesContent = builtins.toJSON { - flannel.iface = "eth1"; - }; - }; - in - { - name = "${rke2.name}-multi-node"; - meta.maintainers = rke2.meta.maintainers; - - nodes = { - server = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.allowedTCPPorts = [ - # Kubernetes API - 6443 - # Canal CNI health checks - 9099 - # RKE2 supervisor API - 9345 - ]; - - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - inherit tokenFile; - inherit agentTokenFile; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - canal-config.content = canalConfig; - # A daemonset that responds 'hello' on port 8000 - network-test.content = { - apiVersion = "apps/v1"; - kind = "DaemonSet"; - metadata = { - name = "test"; - labels.name = "test"; - }; - spec = { - selector.matchLabels.name = "test"; - template = { - metadata.labels.name = "test"; - spec.containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - imagePullPolicy = "Never"; - command = [ - "socat" - "TCP4-LISTEN:8000,fork" - "EXEC:echo hello" - ]; - } - ]; - }; - }; - }; - }; - }; - }; - - agent = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI health checks - networking.firewall.allowedTCPPorts = [ 9099 ]; - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - - # The agent node can work with less resources - virtualisation.memorySize = 2048; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "agent"; - package = rke2; - tokenFile = agentTokenFile; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345"; - nodeIP = config.networking.primaryIPAddress; - manifests.canal-config.content = canalConfig; - images = [ - coreImages - canalImages - helloImage - ]; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - jq = "${pkgs.jq}/bin/jq"; - in - # python - '' - start_all() - - server.wait_for_unit("rke2-server") - agent.wait_for_unit("rke2-agent") - - # Wait for the agent to be ready - server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""") - - server.succeed("${kubectl} cluster-info") - server.wait_until_succeeds("${kubectl} get serviceaccount default") - - # Now verify that each daemonset pod can talk to each other. - server.wait_until_succeeds( - f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]' - ) - - # Get pod IPs - pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines() - pod_ips = [ - server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods - ] - - # Verify each node can ping each pod ip - for pod_ip in pod_ips: - # The CNI sometimes needs a little time - server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - # Verify the server can exec into the pod - for pod in pods: - resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip() - assert resp == "hello", f"Unexpected response from hello daemonset: {resp}" - ''; - } -) diff --git a/nixos/tests/rke2/single-node.nix b/nixos/tests/rke2/single-node.nix deleted file mode 100644 index e19cccf5acc0..000000000000 --- a/nixos/tests/rke2/single-node.nix +++ /dev/null @@ -1,144 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - # A ConfigMap in regular yaml format - cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "from-file"; - data.username = "foo-file"; - }; - in - { - name = "${rke2.name}-single-node"; - meta.maintainers = rke2.meta.maintainers; - nodes.machine = - { - config, - nodes, - pkgs, - ... - }: - { - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - # Slightly reduce resource consumption - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - test-job.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "test"; - spec.template.spec = { - containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - } - ]; - restartPolicy = "Never"; - }; - }; - disabled = { - enable = false; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "disabled"; - data.username = "foo"; - }; - }; - from-file.source = "${cmFile}"; - custom-target = { - enable = true; - target = "my-manifest.json"; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "custom-target"; - data.username = "foo-custom"; - }; - }; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - in - # python - '' - start_all() - - with subtest("Start cluster"): - machine.wait_for_unit("rke2-server") - machine.succeed("${kubectl} cluster-info") - machine.wait_until_succeeds("${kubectl} get serviceaccount default") - - with subtest("Test job completes successfully"): - machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test") - output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip() - assert output == "Hello, world!", f"unexpected output of test job: {output}" - - with subtest("ConfigMap from-file exists"): - output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}" - - with subtest("ConfigMap custom-target exists"): - # Check that the file exists at the custom target path - machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json") - output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}" - - with subtest("Disabled ConfigMap doesn't exist"): - machine.fail("${kubectl} get cm disabled") - ''; - } -) diff --git a/pkgs/applications/networking/cluster/k3s/builder.nix b/pkgs/applications/networking/cluster/k3s/builder.nix index ddb5a5221440..731e1e5955f8 100644 --- a/pkgs/applications/networking/cluster/k3s/builder.nix +++ b/pkgs/applications/networking/cluster/k3s/builder.nix @@ -470,14 +470,11 @@ buildGoModule (finalAttrs: { ; tests = let - mkTests = - version: - let - k3s_version = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor version); - in - lib.mapAttrs (name: value: nixosTests.k3s.${name}.${k3s_version}) nixosTests.k3s; + versionedPackage = "k3s_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor k3sVersion); in - mkTests k3sVersion; + lib.mapAttrs (name: _: nixosTests.k3s.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.k3s + ); imagesList = throw "k3s.imagesList was removed"; airgapImages = throw "k3s.airgapImages was renamed to k3s.airgap-images"; airgapImagesAmd64 = throw "k3s.airgapImagesAmd64 was renamed to k3s.airgap-images-amd64-tar-zst"; diff --git a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md index 580c614bbee3..23eff54b0828 100644 --- a/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md +++ b/pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md @@ -7,7 +7,7 @@ A K3s maintainer, maintains K3s's: - [documentation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/README.md) - [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s) - [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22) -- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s) +- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/rancher) - [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher) - [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating) - updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/) diff --git a/pkgs/applications/networking/cluster/rke2/builder.nix b/pkgs/applications/networking/cluster/rke2/builder.nix index 99bfa58a1d82..63801f9d14f8 100644 --- a/pkgs/applications/networking/cluster/rke2/builder.nix +++ b/pkgs/applications/networking/cluster/rke2/builder.nix @@ -22,6 +22,7 @@ lib: makeWrapper, fetchzip, fetchurl, + versionCheckHook, # Runtime dependencies procps, @@ -42,7 +43,6 @@ lib: # Testing dependencies nixosTests, - testers, }: buildGoModule (finalAttrs: { pname = "rke2"; @@ -129,25 +129,19 @@ buildGoModule (finalAttrs: { go tool nm $out/bin/.rke2-wrapped | grep '_Cfunc__goboringcrypto_' > /dev/null runHook postInstallCheck ''; + nativeInstallCheckInputs = [ versionCheckHook ]; + versionCheckProgramArg = "--version"; passthru = { inherit updateScript; tests = let - moduleTests = - let - package_version = - "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); - in - lib.mapAttrs (name: value: nixosTests.rke2.${name}.${package_version}) nixosTests.rke2; + versionedPackage = + "rke2_" + lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor rke2Version); in - { - version = testers.testVersion { - package = finalAttrs.finalPackage; - version = "v${finalAttrs.version}"; - }; - } - // moduleTests; + lib.mapAttrs (name: _: nixosTests.rke2.${name}.${versionedPackage}) ( + lib.filterAttrs (n: _: n != "all") nixosTests.rke2 + ); } // (lib.mapAttrs (_: value: fetchurl value) imagesVersions);