gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/tests/k3s: generalize to rancher

Browse Source 
Similarly to 2ce16ee67e, prepares for merge with nixos/tests/rke2.
This commit is contained in:
azey
2025-12-11 22:51:36 +01:00
parent bf83174d5a
commit 2f5bd8e63a
15 changed files with 390 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/labeler.yml
+1 -1
View File
@@ -263,7 +263,7 @@
- any-glob-to-any-file:
- nixos/modules/services/cluster/rancher/default.nix
- nixos/modules/services/cluster/rancher/k3s.nix
- nixos/tests/k3s/**/*
- nixos/tests/rancher/**/*
- pkgs/applications/networking/cluster/k3s/**/*
"6.topic: kernel":
nixos/tests/all-tests.nix
+1 -1
View File
@@ -800,7 +800,7 @@ in
jitsi-meet = runTest ./jitsi-meet.nix;
jool = import ./jool.nix { inherit pkgs runTest; };
jotta-cli = runTest ./jotta-cli.nix;
k3s = handleTest ./k3s { };
k3s = handleTest ./rancher { rancherDistro = "k3s"; };
kafka = handleTest ./kafka { };
kanboard = runTest ./web-apps/kanboard.nix;
kanidm = runTest ./kanidm.nix;
nixos/tests/k3s/airgap-images.nix
-34
View File
@@ -1,34 +0,0 @@
# A test that imports k3s airgapped images and verifies that all expected images are present
import ../make-test-python.nix (
{ lib, k3s, ... }:
{
name = "${k3s.name}-airgap-images";
meta.maintainers = lib.teams.k3s.members;
nodes.machine = _: {
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
enable = true;
role = "server";
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
];
images = [ k3s.airgap-images ];
};
};
testScript = ''
machine.wait_for_unit("k3s")
machine.wait_until_succeeds("journalctl -r --no-pager -u k3s | grep \"Imported images from /var/lib/rancher/k3s/agent/images/\"")
'';
}
)
nixos/tests/k3s/auto-deploy.nix
-124
View File
@@ -1,124 +0,0 @@
# Tests whether container images are imported and auto deploying manifests work
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
...
}:
let
pauseImageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = pauseImageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
in
{
name = "${k3s.name}-auto-deploy";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ k3s ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.package = k3s;
# Slightly reduce resource usage
services.k3s.extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
];
services.k3s.images = [
pauseImage
helloImage
];
services.k3s.manifests = {
absent = {
enable = false;
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "absent";
};
};
present = {
target = "foo-namespace.yaml";
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "foo";
};
};
hello.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "hello";
spec = {
template.spec = {
containers = [
{
name = "hello";
image = "test.local/hello:local";
}
];
restartPolicy = "OnFailure";
};
};
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("k3s")
# check existence of the manifest files
machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")
# check if container images got imported
machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'")
# check if resources of manifests got created
machine.wait_until_succeeds("kubectl get ns foo")
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
machine.fail("kubectl get ns absent")
'';
meta.maintainers = lib.teams.k3s.members;
}
)
nixos/tests/k3s/default.nix
-34
View File
@@ -1,34 +0,0 @@
{
system ? builtins.currentSystem,
pkgs ? import ../../.. { inherit system; },
lib ? pkgs.lib,
}:
let
allK3s = lib.filterAttrs (
n: _: lib.strings.hasPrefix "k3s_" n && (builtins.tryEval pkgs.${n}).success
) pkgs;
in
{
airgap-images = lib.mapAttrs (
_: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
) allK3s;
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
auto-deploy-charts = lib.mapAttrs (
_: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
) allK3s;
containerd-config = lib.mapAttrs (
_: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
) allK3s;
etcd = lib.mapAttrs (
_: k3s:
import ./etcd.nix {
inherit system pkgs k3s;
inherit (pkgs) etcd;
}
) allK3s;
kubelet-config = lib.mapAttrs (
_: k3s: import ./kubelet-config.nix { inherit system pkgs k3s; }
) allK3s;
multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
}
nixos/tests/rancher/airgap-images.nix
+38
View File
@@ -0,0 +1,38 @@
# A test that imports k3s airgapped images and verifies that all expected images are present
import ../make-test-python.nix (
{
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
{
name = "${rancherPackage.name}-airgap-images";
meta.maintainers = lib.teams.k3s.members;
nodes.machine = _: {
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
images =
{
k3s = [ rancherPackage.airgap-images ];
}
.${rancherDistro};
};
};
testScript = ''
machine.wait_for_unit("${serviceName}")
machine.wait_until_succeeds("journalctl -r --no-pager -u ${serviceName} | grep \"Imported images from /var/lib/rancher/${rancherDistro}/agent/images/\"")
'';
}
)
nixos/tests/k3s/auto-deploy-charts.nix → nixos/tests/rancher/auto-deploy-charts.nix
+41 -27
View File
@@ -1,15 +1,18 @@
# Tests whether container images are imported and auto deploying Helm charts,
# including the bundled traefik, work
# including the bundled traefik or ingress-nginx, work
import ../make-test-python.nix (
{
k3s,
lib,
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
testImageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
busybox
hello
@@ -24,11 +27,11 @@ import ../make-test-python.nix (
};
# pack the test helm chart as a .tgz archive
package =
pkgs.runCommand "k3s-test-chart.tgz"
pkgs.runCommand "${rancherDistro}-test-chart.tgz"
{
nativeBuildInputs = [ pkgs.kubernetes-helm ];
chart = builtins.toJSON {
name = "k3s-test-chart";
name = "${rancherDistro}-test-chart";
version = "0.1.0";
};
values = builtins.toJSON {
@@ -93,7 +96,7 @@ import ../make-test-python.nix (
};
in
{
name = "${k3s.name}-auto-deploy-helm";
name = "${rancherPackage.name}-auto-deploy-helm";
meta.maintainers = lib.teams.k3s.members;
nodes.machine =
{ pkgs, ... }:
@@ -104,19 +107,23 @@ import ../make-test-python.nix (
diskSize = 4096;
};
environment.systemPackages = [ pkgs.yq-go ];
services.k3s = {
services.${rancherDistro} = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
];
package = rancherPackage;
disable =
{
k3s = lib.remove "traefik" disabledComponents;
rke2 = lib.remove "rke2-ingress-nginx" disabledComponents;
}
.${rancherDistro};
images = [
# Provides the k3s Helm controller
k3s.airgap-images
{
# Provides the k3s Helm controller
k3s = rancherPackage.airgap-images;
rke2 = rancherPackage.images-core-linux-amd64-tar-zst;
}
.${rancherDistro}
testImage
];
autoDeployCharts = {
@@ -133,7 +140,7 @@ import ../make-test-python.nix (
values =
/.
+ builtins.unsafeDiscardStringContext (
builtins.toFile "k3s-test-chart-values.yaml" ''
builtins.toFile "${rancherDistro}-test-chart-values.yaml" ''
runCommand: "echo 'Hello, file!'"
image:
repository: test.local/test
@@ -176,14 +183,14 @@ import ../make-test-python.nix (
''
import json
machine.wait_for_unit("k3s")
machine.wait_for_unit("${serviceName}")
# check existence/absence of chart manifest files
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml")
machine.succeed("test ! -e /var/lib/rancher/${rancherDistro}/server/manifests/disabled.yaml")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/values-file.yaml")
machine.succeed("test -e /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml")
# check that the timeout is set correctly, select only the first doc in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/${rancherDistro}/server/manifests/advanced.yaml"))
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
# wait for test jobs to complete
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
@@ -197,8 +204,15 @@ import ../make-test-python.nix (
t.assertEqual(hello_output.rstrip(), "Hello, world!", "unexpected output of hello job")
t.assertEqual(values_file_output.rstrip(), "Hello, file!", "unexpected output of values file job")
t.assertEqual(advanced_output.rstrip(), "advanced hello", "unexpected output of advanced job")
# wait for bundled traefik deployment
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
# wait for bundled ingress deployment
${
{
k3s = ''
machine.wait_until_succeeds("kubectl -n kube-system rollout status deployment traefik", timeout=180)
'';
}
.${rancherDistro}
}
'';
}
)
nixos/tests/rancher/auto-deploy.nix
+124
View File
@@ -0,0 +1,124 @@
# Tests whether container images are imported and auto deploying manifests work
import ../make-test-python.nix (
{
pkgs,
lib,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
pauseImageEnv = pkgs.buildEnv {
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
busybox
];
};
pauseImage = pkgs.dockerTools.buildImage {
name = "test.local/pause";
tag = "local";
copyToRoot = pauseImageEnv;
config.Entrypoint = [
"/bin/tini"
"--"
"/bin/sleep"
"inf"
];
};
helloImage = pkgs.dockerTools.buildImage {
name = "test.local/hello";
tag = "local";
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
in
{
name = "${rancherPackage.name}-auto-deploy";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = [ rancherPackage ];
# k3s uses enough resources the default vm fails.
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
];
images = [
pauseImage
helloImage
];
manifests = {
absent = {
enable = false;
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "absent";
};
};
present = {
target = "foo-namespace.yaml";
content = {
apiVersion = "v1";
kind = "Namespace";
metadata.name = "foo";
};
};
hello.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "hello";
spec = {
template.spec = {
containers = [
{
name = "hello";
image = "test.local/hello:local";
}
];
restartPolicy = "OnFailure";
};
};
};
};
};
};
testScript = # python
''
start_all()
machine.wait_for_unit("${serviceName}")
# check existence of the manifest files
machine.fail("ls /var/lib/rancher/${rancherDistro}/server/manifests/absent.yaml")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/foo-namespace.yaml")
machine.succeed("ls /var/lib/rancher/${rancherDistro}/server/manifests/hello.yaml")
# check if container images got imported
machine.wait_until_succeeds("crictl img | grep 'test\.local/pause'")
machine.wait_until_succeeds("crictl img | grep 'test\.local/hello'")
# check if resources of manifests got created
machine.wait_until_succeeds("kubectl get ns foo")
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
machine.fail("kubectl get ns absent")
'';
meta.maintainers = lib.teams.k3s.members;
}
)
nixos/tests/k3s/containerd-config.nix → nixos/tests/rancher/containerd-config.nix
+13 -17
View File
@@ -3,14 +3,17 @@ import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
nodeName = "test";
in
{
name = "${k3s.name}-containerd-config";
name = "${rancherPackage.name}-containerd-config";
nodes.machine =
{ ... }:
{
@@ -19,20 +22,13 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
services.${rancherDistro} = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-name ${nodeName}"
];
package = rancherPackage;
disable = disabledComponents;
inherit nodeName;
containerdConfigTemplate = ''
# Base K3s config
# Base ${rancherDistro} config
{{ template "base" . }}
# MAGIC COMMENT
@@ -43,14 +39,14 @@ import ../make-test-python.nix (
testScript = # python
''
start_all()
machine.wait_for_unit("k3s")
machine.wait_for_unit("${serviceName}")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the config template file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl")
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml.tmpl")
t.assertIn("MAGIC COMMENT", out, "the containerd config template does not contain the magic comment")
# test whether the config file contains the magic comment
out=machine.succeed("cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml")
out=machine.succeed("cat /var/lib/rancher/${rancherDistro}/agent/etc/containerd/config.toml")
t.assertIn("MAGIC COMMENT", out, "the containerd config does not contain the magic comment")
'';
nixos/tests/rancher/default.nix
+66
View File
@@ -0,0 +1,66 @@
{
system ? builtins.currentSystem,
pkgs ? import ../../.. { inherit system; },
lib ? pkgs.lib,
# service/package name to test
rancherDistro,
}:
let
allPackages = lib.filterAttrs (
n: pkg: lib.strings.hasPrefix "${rancherDistro}_" n && (builtins.tryEval pkg).success
) pkgs;
mkTestArgs = rancherPackage: {
inherit
system
pkgs
rancherDistro
rancherPackage
;
# systemd service name
serviceName =
{
k3s = "k3s";
rke2 = "rke2-server";
}
.${rancherDistro};
# list passed to services.*.disable,
# for slightly reduced resource usage
disabledComponents =
{
k3s = [
"coredns"
"local-storage"
"metrics-server"
"servicelb"
"traefik"
];
rke2 = [
"rke2-coredns"
"rke2-metrics-server"
"rke2-ingress-nginx"
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
}
.${rancherDistro};
};
importTest =
path: extraArgs: lib.mapAttrs (_: pkg: import path ((mkTestArgs pkg) // extraArgs)) allPackages;
in
{
airgap-images = importTest ./airgap-images.nix { };
auto-deploy = importTest ./auto-deploy.nix { };
auto-deploy-charts = importTest ./auto-deploy-charts.nix { };
containerd-config = importTest ./containerd-config.nix { };
etcd = importTest ./etcd.nix {
inherit (pkgs) etcd;
};
kubelet-config = importTest ./kubelet-config.nix { };
multi-node = importTest ./multi-node.nix { };
single-node = importTest ./single-node.nix { };
}
nixos/tests/k3s/etcd.nix → nixos/tests/rancher/etcd.nix
+18 -19
View File
@@ -3,13 +3,16 @@ import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
etcd,
...
}:
{
name = "${k3s.name}-etcd";
name = "${rancherPackage.name}-etcd";
nodes = {
@@ -39,7 +42,7 @@ import ../make-test-python.nix (
};
};
k3s =
server =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [ jq ];
@@ -47,18 +50,14 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
services.${rancherDistro} = {
enable = true;
role = "server";
package = k3s;
package = rancherPackage;
disable = disabledComponents;
nodeIP = "192.168.1.2";
extraFlags = [
"--datastore-endpoint=\"http://192.168.1.1:2379\""
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-ip 192.168.1.2"
];
};
@@ -95,27 +94,27 @@ import ../make-test-python.nix (
with subtest("should wait for etcdctl endpoint health to succeed"):
etcd.wait_until_succeeds("etcdctl endpoint health")
with subtest("should start k3s"):
k3s.start()
k3s.wait_for_unit("k3s")
with subtest("should start ${rancherDistro}"):
server.start()
server.wait_for_unit("${serviceName}")
with subtest("should test if kubectl works"):
k3s.wait_until_succeeds("k3s kubectl get node")
server.wait_until_succeeds("kubectl get node")
with subtest("should wait for service account to show up; takes a sec"):
k3s.wait_until_succeeds("k3s kubectl get serviceaccount default")
server.wait_until_succeeds("kubectl get serviceaccount default")
with subtest("should create a sample secret object"):
k3s.succeed("k3s kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
server.succeed("kubectl create secret generic nixossecret --from-literal thesecret=abacadabra")
with subtest("should check if secret is correct"):
k3s.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
server.wait_until_succeeds("[[ $(kubectl get secrets nixossecret -o json | jq -r .data.thesecret | base64 -d) == abacadabra ]]")
with subtest("should have a secret in database"):
etcd.wait_until_succeeds("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
with subtest("should delete the secret"):
k3s.succeed("k3s kubectl delete secret nixossecret")
server.succeed("kubectl delete secret nixossecret")
with subtest("should not have a secret in database"):
etcd.wait_until_fails("[[ $(etcdctl get /registry/secrets/default/nixossecret | head -c1 | wc -c) -ne 0 ]]")
nixos/tests/k3s/kubelet-config.nix → nixos/tests/rancher/kubelet-config.nix
+11 -15
View File
@@ -3,7 +3,10 @@ import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
@@ -15,7 +18,7 @@ import ../make-test-python.nix (
containerLogMaxSize = "5Mi";
in
{
name = "${k3s.name}-kubelet-config";
name = "${rancherPackage.name}-kubelet-config";
nodes.machine =
{ pkgs, ... }:
{
@@ -25,18 +28,11 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
services.${rancherDistro} = {
enable = true;
package = k3s;
# Slightly reduce resource usage
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--node-name ${nodeName}"
];
package = rancherPackage;
disable = disabledComponents;
inherit nodeName;
gracefulNodeShutdown = {
enable = true;
inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
@@ -52,11 +48,11 @@ import ../make-test-python.nix (
import json
start_all()
machine.wait_for_unit("k3s")
machine.wait_for_unit("${serviceName}")
# wait until the node is ready
machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
# test whether the kubelet registered an inhibitor lock
machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"")
machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*${rancherDistro}-server.*shutdown\"")
# run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
nixos/tests/k3s/multi-node.nix → nixos/tests/rancher/multi-node.nix
+50 -37
View File
@@ -1,14 +1,17 @@
# A test that runs a multi-node k3s cluster and verify pod networking works across nodes
# A test that runs a multi-node rancher cluster and verifies pod networking works across nodes
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
bashInteractive
@@ -56,11 +59,16 @@ import ../make-test-python.nix (
tokenFile = pkgs.writeText "token" "p@s$w0rd";
in
{
name = "${k3s.name}-multi-node";
name = "${rancherPackage.name}-multi-node";
nodes = {
server =
{ nodes, pkgs, ... }:
{
nodes,
pkgs,
config,
...
}:
{
environment.systemPackages = with pkgs; [
gzip
@@ -70,21 +78,17 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "server";
package = k3s;
package = rancherPackage;
images = [ pauseImage ];
clusterInit = true;
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
"--node-ip ${nodes.server.networking.primaryIPAddress}"
# The interface selection logic of flannel would normally use eth0, as the nixos
# testing driver sets a default route via dev eth0. However, in test setups we
# have to use eth1 for inter-node communication.
@@ -100,7 +104,12 @@ import ../make-test-python.nix (
};
server2 =
{ nodes, pkgs, ... }:
{
nodes,
pkgs,
config,
...
}:
{
environment.systemPackages = with pkgs; [
gzip
@@ -109,21 +118,17 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s = {
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
package = k3s;
package = rancherPackage;
images = [ pauseImage ];
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
clusterInit = false;
nodeIP = config.networking.primaryIPAddress;
disable = disabledComponents;
extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
"--node-ip ${nodes.server2.networking.primaryIPAddress}"
"--flannel-iface eth1"
];
};
@@ -136,20 +141,25 @@ import ../make-test-python.nix (
};
agent =
{ nodes, pkgs, ... }:
{
nodes,
pkgs,
config,
...
}:
{
virtualisation.memorySize = 1024;
virtualisation.diskSize = 2048;
services.k3s = {
services.${rancherDistro} = {
inherit tokenFile;
enable = true;
role = "agent";
package = k3s;
package = rancherPackage;
images = [ pauseImage ];
serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
nodeIP = config.networking.primaryIPAddress;
extraFlags = [
"--pause-image test.local/pause:local"
"--node-ip ${nodes.agent.networking.primaryIPAddress}"
"--flannel-iface eth1"
];
};
@@ -164,25 +174,28 @@ import ../make-test-python.nix (
machines = [server, server2, agent]
for m in machines:
m.wait_for_unit("k3s")
m.wait_for_unit("${serviceName}")
# wait for the agent to show up
server.wait_until_succeeds("k3s kubectl get node agent")
server.wait_until_succeeds("kubectl get node agent")
server.succeed("kubectl get node >&2")
for m in machines:
m.succeed("k3s check-config")
${lib.optionalString (rancherDistro == "k3s") ''
for m in machines:
m.succeed("k3s check-config")
''}
server.succeed("k3s kubectl cluster-info")
server.succeed("kubectl cluster-info")
# Also wait for our service account to show up; it takes a sec
server.wait_until_succeeds("k3s kubectl get serviceaccount default")
server.wait_until_succeeds("kubectl get serviceaccount default")
# Now create a pod on each node via a daemonset and verify they can talk to each other.
server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
server.succeed("kubectl apply -f ${networkTestDaemonset}")
server.wait_until_succeeds(f'[ "$(kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
# Get pod IPs
pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
pods = server.succeed("kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
pod_ips = [server.succeed(f"kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
# Verify each server can ping each pod ip
for pod_ip in pod_ips:
@@ -191,7 +204,7 @@ import ../make-test-python.nix (
agent.succeed(f"ping -c 1 {pod_ip}")
# Verify the pods can talk to each other
for pod in pods:
resp = server.succeed(f"k3s kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
resp = server.succeed(f"kubectl exec {pod} -- socat TCP:{pod_ip}:8000 -")
t.assertEqual(resp.strip(), "server")
'';
nixos/tests/k3s/single-node.nix → nixos/tests/rancher/single-node.nix
+26 -24
View File
@@ -1,14 +1,17 @@
# A test that runs a single node k3s cluster and verify a pod can run
# A test that runs a single node rancher cluster and verifies a pod can run
import ../make-test-python.nix (
{
pkgs,
lib,
k3s,
rancherDistro,
rancherPackage,
serviceName,
disabledComponents,
...
}:
let
imageEnv = pkgs.buildEnv {
name = "k3s-pause-image-env";
name = "${rancherDistro}-pause-image-env";
paths = with pkgs; [
tini
(lib.hiPrio coreutils)
@@ -40,13 +43,13 @@ import ../make-test-python.nix (
'';
in
{
name = "${k3s.name}-single-node";
name = "${rancherPackage.name}-single-node";
nodes.machine =
{ pkgs, ... }:
{
environment.systemPackages = with pkgs; [
k3s
rancherPackage
gzip
];
@@ -54,23 +57,20 @@ import ../make-test-python.nix (
virtualisation.memorySize = 1536;
virtualisation.diskSize = 4096;
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.package = k3s;
# Slightly reduce resource usage
services.k3s.extraFlags = [
"--disable coredns"
"--disable local-storage"
"--disable metrics-server"
"--disable servicelb"
"--disable traefik"
"--pause-image test.local/pause:local"
];
services.${rancherDistro} = {
enable = true;
role = "server";
package = rancherPackage;
disable = disabledComponents;
extraFlags = [
"--pause-image test.local/pause:local"
];
};
users.users = {
noprivs = {
isNormalUser = true;
description = "Can't access k3s by default";
description = "Can't access ${rancherDistro} by default";
password = "*";
};
};
@@ -80,10 +80,12 @@ import ../make-test-python.nix (
''
start_all()
machine.wait_for_unit("k3s")
machine.wait_for_unit("${serviceName}")
machine.succeed("kubectl cluster-info")
machine.fail("sudo -u noprivs kubectl cluster-info")
machine.succeed("k3s check-config")
${lib.optionalString (rancherDistro == "k3s") ''
machine.succeed("k3s check-config")
''}
machine.succeed(
"${pauseImage} | ctr image import -"
)
@@ -95,16 +97,16 @@ import ../make-test-python.nix (
machine.succeed("kubectl delete -f ${testPodYaml}")
# regression test for #176445
machine.fail("journalctl -o cat -u k3s.service | grep 'ipset utility not found'")
machine.fail("journalctl -o cat -u ${serviceName}.service | grep 'ipset utility not found'")
with subtest("Run k3s-killall"):
with subtest("Run ${rancherDistro}-killall"):
# Call the killall script with a clean path to assert that
# all required commands are wrapped
output = machine.succeed("PATH= ${k3s}/bin/k3s-killall.sh 2>&1 | tee /dev/stderr")
output = machine.succeed("PATH= ${rancherPackage}/bin/${rancherDistro}-killall.sh 2>&1 | tee /dev/stderr")
t.assertNotIn("command not found", output, "killall script contains unknown command")
# Check that killall cleaned up properly
machine.fail("systemctl is-active k3s.service")
machine.fail("systemctl is-active ${serviceName}.service")
machine.fail("systemctl list-units | grep containerd")
machine.fail("ip link show | awk -F': ' '{print $2}' | grep -e flannel -e cni0")
machine.fail("ip netns show | grep cni-")
pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md
+1 -1
View File
@@ -7,7 +7,7 @@ A K3s maintainer, maintains K3s's:
- [documentation](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/README.md)
- [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s)
- [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22)
- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s)
- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/rancher)
- [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher)
- [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating)
- updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/)