diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 98bf23d40206..4859fd3b9df6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1346,7 +1346,7 @@ in restic-rest-server = runTest ./restic-rest-server.nix; retroarch = runTest ./retroarch.nix; ringboard = runTest ./ringboard.nix; - rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { }; + rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; }; rkvm = handleTest ./rkvm { }; rmfakecloud = runTest ./rmfakecloud.nix; robustirc-bridge = runTest ./robustirc-bridge.nix; diff --git a/nixos/tests/rancher/airgap-images.nix b/nixos/tests/rancher/airgap-images.nix index 0cb5bfabdd19..cfc6e8da7f13 100644 --- a/nixos/tests/rancher/airgap-images.nix +++ b/nixos/tests/rancher/airgap-images.nix @@ -1,21 +1,21 @@ # A test that imports k3s airgapped images and verifies that all expected images are present import ../make-test-python.nix ( { + pkgs, lib, rancherDistro, rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: { name = "${rancherPackage.name}-airgap-images"; - meta.maintainers = lib.teams.k3s.members; nodes.machine = _: { - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; @@ -23,8 +23,10 @@ import ../make-test-python.nix ( package = rancherPackage; disable = disabledComponents; images = - { + coreImages + ++ { k3s = [ rancherPackage.airgap-images ]; + rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages } .${rancherDistro}; }; @@ -34,5 +36,7 @@ import ../make-test-python.nix ( machine.wait_for_unit("${serviceName}") machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"") ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/auto-deploy-charts.nix b/nixos/tests/rancher/auto-deploy-charts.nix index 848a88c345b9..e0a954b915f6 100644 --- a/nixos/tests/rancher/auto-deploy-charts.nix +++ b/nixos/tests/rancher/auto-deploy-charts.nix @@ -8,6 +8,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -97,16 +99,18 @@ import ../make-test-python.nix ( in { name = "${rancherPackage.name}-auto-deploy-helm"; - meta.maintainers = lib.teams.k3s.members; + nodes.machine = { pkgs, ... }: { - # k3s uses enough resources the default vm fails. - virtualisation = { - memorySize = 1536; - diskSize = 4096; - }; - environment.systemPackages = [ pkgs.yq-go ]; + environment.systemPackages = with pkgs; [ + kubectl + yq-go + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; + services.${rancherDistro} = { enable = true; package = rancherPackage; @@ -116,16 +120,13 @@ import ../make-test-python.nix ( rke2 = lib.remove "rke2-ingress-nginx" disabledComponents; } .${rancherDistro}; - images = [ - { - # Provides the k3s Helm controller - k3s = rancherPackage.airgap-images; - rke2 = rancherPackage.images-core-linux-amd64-tar-zst; - } - .${rancherDistro} - - testImage - ]; + images = + coreImages + # Provides the k3s Helm controller + ++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images + ++ [ + testImage + ]; autoDeployCharts = { # regular test chart that should get installed hello = testChart; @@ -180,17 +181,25 @@ import ../make-test-python.nix ( }; testScript = # python + let + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; + in '' import json machine.wait_for_unit("${serviceName}") # check existence/absence of chart manifest files - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") - machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml") - machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml") - # check that the timeout is set correctly, select only the first doc in advanced.yaml - advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")) + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") + machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}") + machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}") + # check that the timeout is set correctly, select only the first item in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")) t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout") # wait for test jobs to complete machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) @@ -210,9 +219,14 @@ import ../make-test-python.nix ( k3s = '' machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180) ''; + rke2 = '' + machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180) + ''; } .${rancherDistro} } ''; + + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/auto-deploy.nix b/nixos/tests/rancher/auto-deploy.nix index 74784f159cc9..5d3abfff040f 100644 --- a/nixos/tests/rancher/auto-deploy.nix +++ b/nixos/tests/rancher/auto-deploy.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -35,6 +37,13 @@ import ../make-test-python.nix ( copyToRoot = pkgs.hello; config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; }; + + manifestFormat = + { + k3s = "yaml"; + rke2 = "json"; + } + .${rancherDistro}; in { name = "${rancherPackage.name}-auto-deploy"; @@ -42,11 +51,13 @@ import ../make-test-python.nix ( nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ rancherPackage ]; + environment.systemPackages = with pkgs; [ + kubectl + cri-tools + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; @@ -56,7 +67,7 @@ import ../make-test-python.nix ( extraFlags = [ "--pause-image test.local/pause:local" ]; - images = [ + images = coreImages ++ [ pauseImage helloImage ]; @@ -71,7 +82,7 @@ import ../make-test-python.nix ( }; present = { - target = "foo-namespace.yaml"; + target = "foo-namespace.${manifestFormat}"; content = { apiVersion = "v1"; kind = "Namespace"; @@ -105,13 +116,14 @@ import ../make-test-python.nix ( machine.wait_for_unit("${serviceName}") # check existence of the manifest files - machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml") - machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml") + machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}") + machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}") # check if container images got imported - machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'") - machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'") + # for some reason, RKE2 also uses /run/k3s + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'") + machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'") # check if resources of manifests got created machine.wait_until_succeeds("kubectl get ns foo") @@ -119,6 +131,6 @@ import ../make-test-python.nix ( machine.fail("kubectl get ns absent") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/containerd-config.nix b/nixos/tests/rancher/containerd-config.nix index a4b064edad21..f7ab7af2e758 100644 --- a/nixos/tests/rancher/containerd-config.nix +++ b/nixos/tests/rancher/containerd-config.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -17,15 +19,19 @@ import ../make-test-python.nix ( nodes.machine = { ... }: { - environment.systemPackages = [ pkgs.jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; services.${rancherDistro} = { enable = true; package = rancherPackage; disable = disabledComponents; + images = coreImages; inherit nodeName; containerdConfigTemplate = '' # Base ${rancherDistro} config @@ -50,6 +56,6 @@ import ../make-test-python.nix ( t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/default.nix b/nixos/tests/rancher/default.nix index 89586b517bfd..2ca5db896e32 100644 --- a/nixos/tests/rancher/default.nix +++ b/nixos/tests/rancher/default.nix @@ -47,6 +47,43 @@ let ]; } .${rancherDistro}; + + # images that must be present for all tests + coreImages = + { + k3s = [ ]; + + rke2 = + { + aarch64-linux = [ + rancherPackage.images-core-linux-arm64-tar-zst + rancherPackage.images-canal-linux-arm64-tar-zst + ]; + x86_64-linux = [ + rancherPackage.images-core-linux-amd64-tar-zst + rancherPackage.images-canal-linux-amd64-tar-zst + ]; + } + .${pkgs.stdenv.hostPlatform.system} + or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"); + } + .${rancherDistro}; + + # virtualization.* attrs, since all distros + # need more resources than the default + vmResources = + { + k3s = { + memorySize = 1536; + diskSize = 4096; + }; + rke2 = { + cores = 4; + memorySize = 4096; + diskSize = 8092; + }; + } + .${rancherDistro}; }; importTest = @@ -57,9 +94,7 @@ in auto-deploy = importTest ./auto-deploy.nix { }; auto-deploy-charts = importTest ./auto-deploy-charts.nix { }; containerd-config = importTest ./containerd-config.nix { }; - etcd = importTest ./etcd.nix { - inherit (pkgs) etcd; - }; + etcd = importTest ./etcd.nix { }; kubelet-config = importTest ./kubelet-config.nix { }; multi-node = importTest ./multi-node.nix { }; single-node = importTest ./single-node.nix { }; diff --git a/nixos/tests/rancher/etcd.nix b/nixos/tests/rancher/etcd.nix index 2e40472d4911..0bc1941dd51a 100644 --- a/nixos/tests/rancher/etcd.nix +++ b/nixos/tests/rancher/etcd.nix @@ -7,7 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, - etcd, + coreImages, + vmResources, ... }: @@ -45,16 +46,20 @@ import ../make-test-python.nix ( server = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; + + virtualisation = vmResources; services.${rancherDistro} = { enable = true; role = "server"; package = rancherPackage; disable = disabledComponents; + images = coreImages; nodeIP = "192.168.1.2"; extraFlags = [ "--datastore-endpoint=\"http://192.168.1.1:2379\"" @@ -120,6 +125,7 @@ import ../make-test-python.nix ( etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]") ''; - meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members; + meta.maintainers = + pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/kubelet-config.nix b/nixos/tests/rancher/kubelet-config.nix index 322d40e69108..fb921876a584 100644 --- a/nixos/tests/rancher/kubelet-config.nix +++ b/nixos/tests/rancher/kubelet-config.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -22,16 +24,19 @@ import ../make-test-python.nix ( nodes.machine = { pkgs, ... }: { - environment.systemPackages = [ pkgs.jq ]; + environment.systemPackages = with pkgs; [ + kubectl + jq + ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; package = rancherPackage; disable = disabledComponents; + images = coreImages; inherit nodeName; gracefulNodeShutdown = { enable = true; @@ -52,7 +57,7 @@ import ../make-test-python.nix ( # wait until the node is ready machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""") # test whether the kubelet registered an inhibitor lock - machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"") + machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"") # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &") machine.wait_until_succeeds("nc -z 127.0.0.1 8001") @@ -67,6 +72,6 @@ import ../make-test-python.nix ( t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/multi-node.nix b/nixos/tests/rancher/multi-node.nix index 14671a7ae59c..acc077613d64 100644 --- a/nixos/tests/rancher/multi-node.nix +++ b/nixos/tests/rancher/multi-node.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -57,6 +59,13 @@ import ../make-test-python.nix ( command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] ''; tokenFile = pkgs.writeText "token" "p@s$w0rd"; + + supervisorPort = + { + k3s = "6443"; + rke2 = "9345"; + } + .${rancherDistro}; in { name = "${rancherPackage.name}-multi-node"; @@ -71,34 +80,63 @@ import ../make-test-python.nix ( }: { environment.systemPackages = with pkgs; [ + kubectl gzip jq ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - services.${rancherDistro} = { - inherit tokenFile; - enable = true; - role = "server"; - package = rancherPackage; - images = [ pauseImage ]; - clusterInit = true; - nodeIP = config.networking.primaryIPAddress; - disable = disabledComponents; - extraFlags = [ - "--pause-image test.local/pause:local" - # The interface selection logic of flannel would normally use eth0, as the nixos - # testing driver sets a default route via dev eth0. However, in test setups we - # have to use eth1 for inter-node communication. - "--flannel-iface eth1" - ]; - }; + virtualisation = vmResources; + + services.${rancherDistro} = lib.mkMerge [ + { + inherit tokenFile; + enable = true; + role = "server"; + package = rancherPackage; + images = coreImages ++ [ pauseImage ]; + nodeIP = config.networking.primaryIPAddress; + disable = disabledComponents; + extraFlags = [ + "--pause-image test.local/pause:local" + ]; + } + { + k3s = { + clusterInit = true; + extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition + }; + + # The interface selection logic of flannel & canal would normally use eth0, as + # the nixos testing driver sets a default route via dev eth0. However, in test + # setups we have to use eth1 for inter-node communication. + # For K3s this can be handled via --flannel-iface, but RKE2's canal has to be + # configured with this manifest. + rke2.manifests.canal-config.content = { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChartConfig"; + metadata = { + name = "rke2-canal"; + namespace = "kube-system"; + }; + # spec.valuesContent needs to a string, either json or yaml + spec.valuesContent = builtins.toJSON { + flannel.iface = "eth1"; + }; + }; + } + .${rancherDistro} + ]; + + networking.firewall.enable = false; networking.firewall.allowedTCPPorts = [ 2379 2380 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 ]; networking.firewall.allowedUDPPorts = [ 8472 ]; }; @@ -111,31 +149,32 @@ import ../make-test-python.nix ( ... }: { - environment.systemPackages = with pkgs; [ - gzip - jq - ]; - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { inherit tokenFile; enable = true; + role = "server"; package = rancherPackage; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443"; - clusterInit = false; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; disable = disabledComponents; extraFlags = [ "--pause-image test.local/pause:local" - "--flannel-iface eth1" - ]; + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; + + networking.firewall.enable = false; networking.firewall.allowedTCPPorts = [ 2379 2380 6443 + ] + ++ lib.optionals (rancherDistro == "rke2") [ + 9099 + 9345 ]; networking.firewall.allowedUDPPorts = [ 8472 ]; }; @@ -148,22 +187,23 @@ import ../make-test-python.nix ( ... }: { - virtualisation.memorySize = 1024; - virtualisation.diskSize = 2048; + virtualisation = vmResources; + services.${rancherDistro} = { inherit tokenFile; enable = true; role = "agent"; package = rancherPackage; - images = [ pauseImage ]; - serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443"; + images = coreImages ++ [ pauseImage ]; + serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}"; nodeIP = config.networking.primaryIPAddress; extraFlags = [ "--pause-image test.local/pause:local" - "--flannel-iface eth1" - ]; + ] + ++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1"; }; - networking.firewall.allowedTCPPorts = [ 6443 ]; + + networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099; networking.firewall.allowedUDPPorts = [ 8472 ]; }; }; @@ -172,13 +212,12 @@ import ../make-test-python.nix ( '' start_all() - machines = [server, server2, agent] - for m in machines: + servers = [server, server2] + for m in servers: m.wait_for_unit("${serviceName}") # wait for the agent to show up server.wait_until_succeeds("kubectl get node agent") - server.succeed("kubectl get node >&2") ${lib.optionalString (rancherDistro == "k3s") '' for m in machines: @@ -208,6 +247,6 @@ import ../make-test-python.nix ( t.assertEqual(resp.strip(), "server") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rancher/single-node.nix b/nixos/tests/rancher/single-node.nix index d894b951d798..57c1691afdee 100644 --- a/nixos/tests/rancher/single-node.nix +++ b/nixos/tests/rancher/single-node.nix @@ -7,6 +7,8 @@ import ../make-test-python.nix ( rancherPackage, serviceName, disabledComponents, + coreImages, + vmResources, ... }: let @@ -18,7 +20,7 @@ import ../make-test-python.nix ( busybox ]; }; - pauseImage = pkgs.dockerTools.streamLayeredImage { + pauseImage = pkgs.dockerTools.buildLayeredImage { name = "test.local/pause"; tag = "local"; contents = imageEnv; @@ -29,7 +31,7 @@ import ../make-test-python.nix ( "inf" ]; }; - testPodYaml = pkgs.writeText "test.yml" '' + testPodYaml = pkgs.writeText "test.yaml" '' apiVersion: v1 kind: Pod metadata: @@ -46,22 +48,22 @@ import ../make-test-python.nix ( name = "${rancherPackage.name}-single-node"; nodes.machine = - { pkgs, ... }: + { config, pkgs, ... }: { environment.systemPackages = with pkgs; [ - rancherPackage + kubectl gzip ]; + environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml"; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = 1536; - virtualisation.diskSize = 4096; + virtualisation = vmResources; services.${rancherDistro} = { enable = true; role = "server"; package = rancherPackage; disable = disabledComponents; + images = coreImages ++ [ pauseImage ]; extraFlags = [ "--pause-image test.local/pause:local" ]; @@ -86,14 +88,11 @@ import ../make-test-python.nix ( ${lib.optionalString (rancherDistro == "k3s") '' machine.succeed("k3s check-config") ''} - machine.succeed( - "${pauseImage} | ctr image import -" - ) # Also wait for our service account to show up; it takes a sec machine.wait_until_succeeds("kubectl get serviceaccount default") machine.succeed("kubectl apply -f ${testPodYaml}") - machine.succeed("kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s") machine.succeed("kubectl delete -f ${testPodYaml}") # regression test for #176445 @@ -107,11 +106,11 @@ import ../make-test-python.nix ( # Check that killall cleaned up properly machine.fail("systemctl is-active ${serviceName}.service") - machine.fail("systemctl list-units | grep containerd") + machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5) machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0") machine.fail("ip netns show | grep cni-") ''; - meta.maintainers = lib.teams.k3s.members; + meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers; } ) diff --git a/nixos/tests/rke2/default.nix b/nixos/tests/rke2/default.nix deleted file mode 100644 index 90415a262acf..000000000000 --- a/nixos/tests/rke2/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - system ? builtins.currentSystem, - pkgs ? import ../../.. { inherit system; }, - lib ? pkgs.lib, -}: -let - allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs; -in -{ - # Run a single node rke2 cluster and verify a pod can run - singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2; - # Run a multi-node rke2 cluster and verify pod networking works across nodes - multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2; -} diff --git a/nixos/tests/rke2/multi-node.nix b/nixos/tests/rke2/multi-node.nix deleted file mode 100644 index ee394c07f828..000000000000 --- a/nixos/tests/rke2/multi-node.nix +++ /dev/null @@ -1,207 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.buildEnv { - name = "rke2-hello-image-env"; - paths = with pkgs; [ - coreutils - socat - ]; - }; - }; - tokenFile = pkgs.writeText "token" "p@s$w0rd"; - agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd"; - # Let flannel use eth1 to enable inter-node communication in tests - canalConfig = { - apiVersion = "helm.cattle.io/v1"; - kind = "HelmChartConfig"; - metadata = { - name = "rke2-canal"; - namespace = "kube-system"; - }; - # spec.valuesContent needs to a string, either json or yaml - spec.valuesContent = builtins.toJSON { - flannel.iface = "eth1"; - }; - }; - in - { - name = "${rke2.name}-multi-node"; - meta.maintainers = rke2.meta.maintainers; - - nodes = { - server = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - networking.firewall.allowedTCPPorts = [ - # Kubernetes API - 6443 - # Canal CNI health checks - 9099 - # RKE2 supervisor API - 9345 - ]; - - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - inherit tokenFile; - inherit agentTokenFile; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - canal-config.content = canalConfig; - # A daemonset that responds 'hello' on port 8000 - network-test.content = { - apiVersion = "apps/v1"; - kind = "DaemonSet"; - metadata = { - name = "test"; - labels.name = "test"; - }; - spec = { - selector.matchLabels.name = "test"; - template = { - metadata.labels.name = "test"; - spec.containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - imagePullPolicy = "Never"; - command = [ - "socat" - "TCP4-LISTEN:8000,fork" - "EXEC:echo hello" - ]; - } - ]; - }; - }; - }; - }; - }; - }; - - agent = - { - config, - nodes, - pkgs, - ... - }: - { - # Canal CNI health checks - networking.firewall.allowedTCPPorts = [ 9099 ]; - # Canal CNI with VXLAN - networking.firewall.allowedUDPPorts = [ 8472 ]; - - # The agent node can work with less resources - virtualisation.memorySize = 2048; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "agent"; - package = rke2; - tokenFile = agentTokenFile; - serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345"; - nodeIP = config.networking.primaryIPAddress; - manifests.canal-config.content = canalConfig; - images = [ - coreImages - canalImages - helloImage - ]; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - jq = "${pkgs.jq}/bin/jq"; - in - # python - '' - start_all() - - server.wait_for_unit("rke2-server") - agent.wait_for_unit("rke2-agent") - - # Wait for the agent to be ready - server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""") - - server.succeed("${kubectl} cluster-info") - server.wait_until_succeeds("${kubectl} get serviceaccount default") - - # Now verify that each daemonset pod can talk to each other. - server.wait_until_succeeds( - f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]' - ) - - # Get pod IPs - pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines() - pod_ips = [ - server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods - ] - - # Verify each node can ping each pod ip - for pod_ip in pod_ips: - # The CNI sometimes needs a little time - server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5) - # Verify the server can exec into the pod - for pod in pods: - resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip() - assert resp == "hello", f"Unexpected response from hello daemonset: {resp}" - ''; - } -) diff --git a/nixos/tests/rke2/single-node.nix b/nixos/tests/rke2/single-node.nix deleted file mode 100644 index e19cccf5acc0..000000000000 --- a/nixos/tests/rke2/single-node.nix +++ /dev/null @@ -1,144 +0,0 @@ -import ../make-test-python.nix ( - { - pkgs, - lib, - rke2, - ... - }: - let - throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}"; - coreImages = - { - aarch64-linux = rke2.images-core-linux-arm64-tar-zst; - x86_64-linux = rke2.images-core-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - canalImages = - { - aarch64-linux = rke2.images-canal-linux-arm64-tar-zst; - x86_64-linux = rke2.images-canal-linux-amd64-tar-zst; - } - .${pkgs.stdenv.hostPlatform.system} or throwSystem; - helloImage = pkgs.dockerTools.buildImage { - name = "test.local/hello"; - tag = "local"; - compressor = "zstd"; - copyToRoot = pkgs.hello; - config.Entrypoint = [ "${pkgs.hello}/bin/hello" ]; - }; - # A ConfigMap in regular yaml format - cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "from-file"; - data.username = "foo-file"; - }; - in - { - name = "${rke2.name}-single-node"; - meta.maintainers = rke2.meta.maintainers; - nodes.machine = - { - config, - nodes, - pkgs, - ... - }: - { - # RKE2 needs more resources than the default - virtualisation.cores = 4; - virtualisation.memorySize = 4096; - virtualisation.diskSize = 8092; - - services.rke2 = { - enable = true; - role = "server"; - package = rke2; - # Without nodeIP the apiserver starts with the wrong service IP family - nodeIP = config.networking.primaryIPAddress; - # Slightly reduce resource consumption - disable = [ - "rke2-coredns" - "rke2-metrics-server" - "rke2-ingress-nginx" - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" - ]; - images = [ - coreImages - canalImages - helloImage - ]; - manifests = { - test-job.content = { - apiVersion = "batch/v1"; - kind = "Job"; - metadata.name = "test"; - spec.template.spec = { - containers = [ - { - name = "hello"; - image = "${helloImage.imageName}:${helloImage.imageTag}"; - } - ]; - restartPolicy = "Never"; - }; - }; - disabled = { - enable = false; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "disabled"; - data.username = "foo"; - }; - }; - from-file.source = "${cmFile}"; - custom-target = { - enable = true; - target = "my-manifest.json"; - content = { - apiVersion = "v1"; - kind = "ConfigMap"; - metadata.name = "custom-target"; - data.username = "foo-custom"; - }; - }; - }; - }; - }; - - testScript = - let - kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; - in - # python - '' - start_all() - - with subtest("Start cluster"): - machine.wait_for_unit("rke2-server") - machine.succeed("${kubectl} cluster-info") - machine.wait_until_succeeds("${kubectl} get serviceaccount default") - - with subtest("Test job completes successfully"): - machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test") - output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip() - assert output == "Hello, world!", f"unexpected output of test job: {output}" - - with subtest("ConfigMap from-file exists"): - output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}" - - with subtest("ConfigMap custom-target exists"): - # Check that the file exists at the custom target path - machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json") - output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip() - assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}" - - with subtest("Disabled ConfigMap doesn't exist"): - machine.fail("${kubectl} get cm disabled") - ''; - } -)