gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/tests/rke2: merge into nixos/tests/k3s

Browse Source 
Follow-up on ac4abf7f7b.
This commit is contained in:
azey
2025-12-12 17:15:50 +01:00
parent 2f5bd8e63a
commit e963fce108
13 changed files with 236 additions and 481 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/tests/all-tests.nix
+1 -1
View File
@@ -1346,7 +1346,7 @@ in
restic-rest-server = runTest ./restic-rest-server.nix;
retroarch = runTest ./retroarch.nix;
ringboard = runTest ./ringboard.nix;
rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rke2 { };
rke2 = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./rancher { rancherDistro = "rke2"; };
rkvm = handleTest ./rkvm { };
rmfakecloud = runTest ./rmfakecloud.nix;
robustirc-bridge = runTest ./robustirc-bridge.nix;
nixos/tests/rancher/airgap-images.nix
+9 -5
View File
@@ -1,21 +1,21 @@
# A test that imports k3s airgapped images and verifies that all expected images are present
import ../make-test-python.nix (
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
{
name = "${rancherPackage.name}-airgap-images";
meta.maintainers = lib.teams.k3s.members;
nodes.machine = _: {
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
@@ -23,8 +23,10 @@ import ../make-test-python.nix (
package = rancherPackage;
disable = disabledComponents;
images =
{
coreImages
++ {
k3s = [ rancherPackage.airgap-images ];
rke2 = [ ]; # RKE2 already includes its airgap-images in coreImages
}
.${rancherDistro};
};
@@ -34,5 +36,7 @@ import ../make-test-python.nix (
machine.wait_for_unit("${serviceName}")
machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/auto-deploy-charts.nix
+37 -23
View File
@@ -8,6 +8,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -97,16 +99,18 @@ import ../make-test-python.nix (
in
{
name = "${rancherPackage.name}-auto-deploy-helm";
meta.maintainers = lib.teams.k3s.members;
nodes.machine =
{ pkgs, ... }:
{
# k3s uses enough resources the default vm fails.
virtualisation = {
memorySize = 1536;
diskSize = 4096;
};
environment.systemPackages = [ pkgs.yq-go ];
environment.systemPackages = with pkgs; [
kubectl
yq-go
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
@@ -116,16 +120,13 @@ import ../make-test-python.nix (
rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;
}
.${rancherDistro};
images = [
{
# Provides the k3s Helm controller
k3s = rancherPackage.airgap-images;
rke2 = rancherPackage.images-core-linux-amd64-tar-zst;
}
.${rancherDistro}
testImage
];
images =
coreImages
# Provides the k3s Helm controller
++ lib.optional (rancherDistro == "k3s") rancherPackage.airgap-images
++ [
testImage
];
autoDeployCharts = {
# regular test chart that should get installed
hello = testChart;
@@ -180,17 +181,25 @@ import ../make-test-python.nix (
};
testScript = # python
let
manifestFormat =
{
k3s = "yaml";
rke2 = "json";
}
.${rancherDistro};
in
''
import json
machine.wait_for_unit("${serviceName}")
# check existence/absence of chart manifest files
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml")
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")
# check that the timeout is set correctly, select only the first doc in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml"))
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.${manifestFormat}")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.${manifestFormat}")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}")
# check that the timeout is set correctly, select only the first item in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.${manifestFormat}"))
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
# wait for test jobs to complete
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
@@ -210,9 +219,14 @@ import ../make-test-python.nix (
k3s = ''
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
'';
rke2 = ''
machine.wait_until_succeeds("kubectl -n kube-system rollout status daemonset rke2-ingress-nginx-controller", timeout=180)
'';
}
.${rancherDistro}
}
'';
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/auto-deploy.nix
+24 -12
View File
@@ -7,6 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -35,6 +37,13 @@ import ../make-test-python.nix (
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
manifestFormat =
{
k3s = "yaml";
rke2 = "json";
}
.${rancherDistro};
in
{
name = "${rancherPackage.name}-auto-deploy";
@@ -42,11 +51,13 @@ import ../make-test-python.nix (
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ rancherPackage ];
environment.systemPackages = with pkgs; [
kubectl
cri-tools
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
@@ -56,7 +67,7 @@ import ../make-test-python.nix (
extraFlags = [
"--pause-image test.local/pause:local"
];
images = [
images = coreImages ++ [
pauseImage
helloImage
];
@@ -71,7 +82,7 @@ import ../make-test-python.nix (
};
present = {
target = "foo-namespace.yaml";
target = "foo-namespace.${manifestFormat}";
content = {
apiVersion = "v1";
kind = "Namespace";
@@ -105,13 +116,14 @@ import ../make-test-python.nix (
machine.wait_for_unit("${serviceName}")
# check existence of the manifest files
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml")
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.${manifestFormat}")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.${manifestFormat}")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.${manifestFormat}")
# check if container images got imported
machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'")
# for some reason, RKE2 also uses /run/k3s
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl -r /run/k3s/containerd/containerd.sock img | grep 'test\.local/hello'")
# check if resources of manifests got created
machine.wait_until_succeeds("kubectl get ns foo")
@@ -119,6 +131,6 @@ import ../make-test-python.nix (
machine.fail("kubectl get ns absent")
'';
meta.maintainers = lib.teams.k3s.members;
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/containerd-config.nix
+11 -5
View File
@@ -7,6 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -17,15 +19,19 @@ import ../make-test-python.nix (
nodes.machine =
{ ... }:
{
environment.systemPackages = [ pkgs.jq ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
inherit nodeName;
containerdConfigTemplate = ''
# Base ${rancherDistro} config
@@ -50,6 +56,6 @@ import ../make-test-python.nix (
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
'';
meta.maintainers = lib.teams.k3s.members;
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/default.nix
+38 -3
View File
@@ -47,6 +47,43 @@ let
];
}
.${rancherDistro};
# images that must be present for all tests
coreImages =
{
k3s = [ ];
rke2 =
{
aarch64-linux = [
rancherPackage.images-core-linux-arm64-tar-zst
rancherPackage.images-canal-linux-arm64-tar-zst
];
x86_64-linux = [
rancherPackage.images-core-linux-amd64-tar-zst
rancherPackage.images-canal-linux-amd64-tar-zst
];
}
.${pkgs.stdenv.hostPlatform.system}
or (throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}");
}
.${rancherDistro};
# virtualization.* attrs, since all distros
# need more resources than the default
vmResources =
{
k3s = {
memorySize = 1536;
diskSize = 4096;
};
rke2 = {
cores = 4;
memorySize = 4096;
diskSize = 8092;
};
}
.${rancherDistro};
};
importTest =
@@ -57,9 +94,7 @@ in
auto-deploy = importTest ./auto-deploy.nix { };
auto-deploy-charts = importTest ./auto-deploy-charts.nix { };
containerd-config = importTest ./containerd-config.nix { };
etcd = importTest ./etcd.nix {
inherit (pkgs) etcd;
};
etcd = importTest ./etcd.nix { };
kubelet-config = importTest ./kubelet-config.nix { };
multi-node = importTest ./multi-node.nix { };
single-node = importTest ./single-node.nix { };
nixos/tests/rancher/etcd.nix
+12 -6
View File
@@ -7,7 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
etcd,
coreImages,
vmResources,
...
}:
@@ -45,16 +46,20 @@ import ../make-test-python.nix (
server =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [ jq ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
nodeIP = "192.168.1.2";
extraFlags = [
"--datastore-endpoint=\"http://192.168.1.1:2379\""
@@ -120,6 +125,7 @@ import ../make-test-python.nix (
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
'';
meta.maintainers = etcd.meta.maintainers ++ lib.teams.k3s.members;
meta.maintainers =
pkgs.etcd.meta.maintainers ++ lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/kubelet-config.nix
+11 -6
View File
@@ -7,6 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -22,16 +24,19 @@ import ../make-test-python.nix (
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
environment.systemPackages = with pkgs; [
kubectl
jq
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
package = rancherPackage;
disable = disabledComponents;
images = coreImages;
inherit nodeName;
gracefulNodeShutdown = {
enable = true;
@@ -52,7 +57,7 @@ import ../make-test-python.nix (
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the kubelet registered an inhibitor lock
machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"")
machine.succeed("systemd-inhibit --list --no-legend | grep \"^kubelet.*shutdown\"")
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
@@ -67,6 +72,6 @@ import ../make-test-python.nix (
t.assertEqual(kubelet_config["containerLogMaxSize"],"${containerLogMaxSize}")
'';
meta.maintainers = lib.teams.k3s.members;
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/multi-node.nix
+81 -42
View File
@@ -7,6 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -57,6 +59,13 @@ import ../make-test-python.nix (
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
'';
tokenFile = pkgs.writeText "token" "p@s$w0rd";
supervisorPort =
{
k3s = "6443";
rke2 = "9345";
}
.${rancherDistro};
in
{
name = "${rancherPackage.name}-multi-node";
@@ -71,34 +80,63 @@ import ../make-test-python.nix (
}:
{
environment.systemPackages = with pkgs; [
kubectl
gzip
jq
];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "server";
package = rancherPackage;
images = [ pauseImage ];
clusterInit = true;
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
# The interface selection logic of flannel would normally use eth0, as the nixos
# testing driver sets a default route via dev eth0. However, in test setups we
# have to use eth1 for inter-node communication.
"--flannel-iface eth1"
];
};
virtualisation = vmResources;
services.${rancherDistro} = lib.mkMerge [
{
inherit tokenFile;
enable = true;
role = "server";
package = rancherPackage;
images = coreImages ++ [ pauseImage ];
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
];
}
{
k3s = {
clusterInit = true;
extraFlags = [ "--flannel-iface eth1" ]; # see canalConfig definition
};
# The interface selection logic of flannel & canal would normally use eth0, as
# the nixos testing driver sets a default route via dev eth0. However, in test
# setups we have to use eth1 for inter-node communication.
# For K3s this can be handled via --flannel-iface, but RKE2's canal has to be
# configured with this manifest.
rke2.manifests.canal-config.content = {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChartConfig";
metadata = {
name = "rke2-canal";
namespace = "kube-system";
};
# spec.valuesContent needs to a string, either json or yaml
spec.valuesContent = builtins.toJSON {
flannel.iface = "eth1";
};
};
}
.${rancherDistro}
];
networking.firewall.enable = false;
networking.firewall.allowedTCPPorts = [
2379
2380
6443
]
++ lib.optionals (rancherDistro == "rke2") [
9099
9345
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
@@ -111,31 +149,32 @@ import ../make-test-python.nix (
...
}:
{
environment.systemPackages = with pkgs; [
gzip
jq
];
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
virtualisation = vmResources;
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "server";
package = rancherPackage;
images = [ pauseImage ];
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
clusterInit = false;
images = coreImages ++ [ pauseImage ];
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:${supervisorPort}";
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
"--flannel-iface eth1"
];
]
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
};
networking.firewall.enable = false;
networking.firewall.allowedTCPPorts = [
2379
2380
6443
]
++ lib.optionals (rancherDistro == "rke2") [
9099
9345
];
networking.firewall.allowedUDPPorts = [ 8472 ];
};
@@ -148,22 +187,23 @@ import ../make-test-python.nix (
...
}:
{
virtualisation.memorySize = 1024;
virtualisation.diskSize = 2048;
virtualisation = vmResources;
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "agent";
package = rancherPackage;
images = [ pauseImage ];
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
images = coreImages ++ [ pauseImage ];
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:${supervisorPort}";
nodeIP = config.networking.primaryIPAddress;
extraFlags = [
"--pause-image test.local/pause:local"
"--flannel-iface eth1"
];
]
++ lib.optional (rancherDistro == "k3s") "--flannel-iface eth1";
};
networking.firewall.allowedTCPPorts = [ 6443 ];
networking.firewall.allowedTCPPorts = lib.optional (rancherDistro == "rke2") 9099;
networking.firewall.allowedUDPPorts = [ 8472 ];
};
};
@@ -172,13 +212,12 @@ import ../make-test-python.nix (
''
start_all()
machines = [server, server2, agent]
for m in machines:
servers = [server, server2]
for m in servers:
m.wait_for_unit("${serviceName}")
# wait for the agent to show up
server.wait_until_succeeds("kubectl get node agent")
server.succeed("kubectl get node >&2")
${lib.optionalString (rancherDistro == "k3s") ''
for m in machines:
@@ -208,6 +247,6 @@ import ../make-test-python.nix (
t.assertEqual(resp.strip(), "server")
'';
meta.maintainers = lib.teams.k3s.members;
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rancher/single-node.nix
+12 -13
View File
@@ -7,6 +7,8 @@ import ../make-test-python.nix (
rancherPackage,
serviceName,
disabledComponents,
coreImages,
vmResources,
...
}:
let
@@ -18,7 +20,7 @@ import ../make-test-python.nix (
busybox
];
};
pauseImage = pkgs.dockerTools.streamLayeredImage {
pauseImage = pkgs.dockerTools.buildLayeredImage {
name = "test.local/pause";
tag = "local";
contents = imageEnv;
@@ -29,7 +31,7 @@ import ../make-test-python.nix (
"inf"
];
};
testPodYaml = pkgs.writeText "test.yml" ''
testPodYaml = pkgs.writeText "test.yaml" ''
apiVersion: v1
kind: Pod
metadata:
@@ -46,22 +48,22 @@ import ../make-test-python.nix (
name = "${rancherPackage.name}-single-node";
nodes.machine =
{ pkgs, ... }:
{ config, pkgs, ... }:
{
environment.systemPackages = with pkgs; [
rancherPackage
kubectl
gzip
];
environment.sessionVariables.KUBECONFIG = "/etc/rancher/${rancherDistro}/${rancherDistro}.yaml";
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
virtualisation = vmResources;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images = coreImages ++ [ pauseImage ];
extraFlags = [
"--pause-image test.local/pause:local"
];
@@ -86,14 +88,11 @@ import ../make-test-python.nix (
${lib.optionalString (rancherDistro == "k3s") ''
machine.succeed("k3s check-config")
''}
machine.succeed(
"${pauseImage} | ctr image import -"
)
# Also wait for our service account to show up; it takes a sec
machine.wait_until_succeeds("kubectl get serviceaccount default")
machine.succeed("kubectl apply -f ${testPodYaml}")
machine.succeed("kubectl wait --for 'condition=Ready' pod/test")
machine.succeed("kubectl wait --for 'condition=Ready' pod/test --timeout=180s")
machine.succeed("kubectl delete -f ${testPodYaml}")
# regression test for #176445
@@ -107,11 +106,11 @@ import ../make-test-python.nix (
# Check that killall cleaned up properly
machine.fail("systemctl is-active ${serviceName}.service")
machine.fail("systemctl list-units | grep containerd")
machine.wait_until_fails("systemctl list-units | grep containerd", timeout=5)
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
machine.fail("ip netns show | grep cni-")
'';
meta.maintainers = lib.teams.k3s.members;
meta.maintainers = lib.teams.k3s.members ++ pkgs.rke2.meta.maintainers;
}
)
nixos/tests/rke2/default.nix
-14
View File
@@ -1,14 +0,0 @@
{
system ? builtins.currentSystem,
pkgs ? import ../../.. { inherit system; },
lib ? pkgs.lib,
}:
let
allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs;
in
{
# Run a single node rke2 cluster and verify a pod can run
singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2;
# Run a multi-node rke2 cluster and verify pod networking works across nodes
multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2;
}
nixos/tests/rke2/multi-node.nix
-207
View File
@@ -1,207 +0,0 @@
import ../make-test-python.nix (
{
pkgs,
lib,
rke2,
...
}:
let
throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
coreImages =
{
aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
canalImages =
{
aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
compressor = "zstd";
copyToRoot = pkgs.buildEnv {
name = "rke2-hello-image-env";
paths = with pkgs; [
coreutils
socat
];
};
};
tokenFile = pkgs.writeText "token" "p@s$w0rd";
agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";
# Let flannel use eth1 to enable inter-node communication in tests
canalConfig = {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChartConfig";
metadata = {
name = "rke2-canal";
namespace = "kube-system";
};
# spec.valuesContent needs to a string, either json or yaml
spec.valuesContent = builtins.toJSON {
flannel.iface = "eth1";
};
};
in
{
name = "${rke2.name}-multi-node";
meta.maintainers = rke2.meta.maintainers;
nodes = {
server =
{
config,
nodes,
pkgs,
...
}:
{
# Canal CNI with VXLAN
networking.firewall.allowedUDPPorts = [ 8472 ];
networking.firewall.allowedTCPPorts = [
# Kubernetes API
6443
# Canal CNI health checks
9099
# RKE2 supervisor API
9345
];
# RKE2 needs more resources than the default
virtualisation.cores = 4;
virtualisation.memorySize = 4096;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "server";
package = rke2;
inherit tokenFile;
inherit agentTokenFile;
# Without nodeIP the apiserver starts with the wrong service IP family
nodeIP = config.networking.primaryIPAddress;
disable = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
canal-config.content = canalConfig;
# A daemonset that responds 'hello' on port 8000
network-test.content = {
apiVersion = "apps/v1";
kind = "DaemonSet";
metadata = {
name = "test";
labels.name = "test";
};
spec = {
selector.matchLabels.name = "test";
template = {
metadata.labels.name = "test";
spec.containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
imagePullPolicy = "Never";
command = [
"socat"
"TCP4-LISTEN:8000,fork"
"EXEC:echo hello"
];
}
];
};
};
};
};
};
};
agent =
{
config,
nodes,
pkgs,
...
}:
{
# Canal CNI health checks
networking.firewall.allowedTCPPorts = [ 9099 ];
# Canal CNI with VXLAN
networking.firewall.allowedUDPPorts = [ 8472 ];
# The agent node can work with less resources
virtualisation.memorySize = 2048;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "agent";
package = rke2;
tokenFile = agentTokenFile;
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";
nodeIP = config.networking.primaryIPAddress;
manifests.canal-config.content = canalConfig;
images = [
coreImages
canalImages
helloImage
];
};
};
};
testScript =
let
kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
jq = "${pkgs.jq}/bin/jq";
in
# python
''
start_all()
server.wait_for_unit("rke2-server")
agent.wait_for_unit("rke2-agent")
# Wait for the agent to be ready
server.wait_until_succeeds(r"""${kubectl} wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' nodes/agent""")
server.succeed("${kubectl} cluster-info")
server.wait_until_succeeds("${kubectl} get serviceaccount default")
# Now verify that each daemonset pod can talk to each other.
server.wait_until_succeeds(
f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'
)
# Get pod IPs
pods = server.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines()
pod_ips = [
server.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods
]
# Verify each node can ping each pod ip
for pod_ip in pod_ips:
# The CNI sometimes needs a little time
server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
# Verify the server can exec into the pod
for pod in pods:
resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip()
assert resp == "hello", f"Unexpected response from hello daemonset: {resp}"
'';
}
)
nixos/tests/rke2/single-node.nix
-144
View File
@@ -1,144 +0,0 @@
import ../make-test-python.nix (
{
pkgs,
lib,
rke2,
...
}:
let
throwSystem = throw "RKE2: Unsupported system: ${pkgs.stdenv.hostPlatform.system}";
coreImages =
{
aarch64-linux = rke2.images-core-linux-arm64-tar-zst;
x86_64-linux = rke2.images-core-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
canalImages =
{
aarch64-linux = rke2.images-canal-linux-arm64-tar-zst;
x86_64-linux = rke2.images-canal-linux-amd64-tar-zst;
}
.${pkgs.stdenv.hostPlatform.system} or throwSystem;
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
compressor = "zstd";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
# A ConfigMap in regular yaml format
cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "from-file";
data.username = "foo-file";
};
in
{
name = "${rke2.name}-single-node";
meta.maintainers = rke2.meta.maintainers;
nodes.machine =
{
config,
nodes,
pkgs,
...
}:
{
# RKE2 needs more resources than the default
virtualisation.cores = 4;
virtualisation.memorySize = 4096;
virtualisation.diskSize = 8092;
services.rke2 = {
enable = true;
role = "server";
package = rke2;
# Without nodeIP the apiserver starts with the wrong service IP family
nodeIP = config.networking.primaryIPAddress;
# Slightly reduce resource consumption
disable = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
test-job.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "test";
spec.template.spec = {
containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
}
];
restartPolicy = "Never";
};
};
disabled = {
enable = false;
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "disabled";
data.username = "foo";
};
};
from-file.source = "${cmFile}";
custom-target = {
enable = true;
target = "my-manifest.json";
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "custom-target";
data.username = "foo-custom";
};
};
};
};
};
testScript =
let
kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml";
in
# python
''
start_all()
with subtest("Start cluster"):
machine.wait_for_unit("rke2-server")
machine.succeed("${kubectl} cluster-info")
machine.wait_until_succeeds("${kubectl} get serviceaccount default")
with subtest("Test job completes successfully"):
machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test")
output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip()
assert output == "Hello, world!", f"unexpected output of test job: {output}"
with subtest("ConfigMap from-file exists"):
output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}"
with subtest("ConfigMap custom-target exists"):
# Check that the file exists at the custom target path
machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json")
output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}"
with subtest("Disabled ConfigMap doesn't exist"):
machine.fail("${kubectl} get cm disabled")
'';
}
)