nixos/k3s: extend k3s module
This contribution extends the k3s module to enable the usage of Helm charts and container images in air-gapped environments. Additionally, the manifests option allows to specify arbitrary manifests that are deployed by k3s automatically. It is now possible to deploy Kubernetes workloads using the k3s module.
This commit is contained in:
Robert Rose
@@ -17,6 +17,109 @@ let
|
||||
]
|
||||
++ config
|
||||
) instruction;
|
||||
|
||||
manifestDir = "/var/lib/rancher/k3s/server/manifests";
|
||||
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
||||
imageDir = "/var/lib/rancher/k3s/agent/images";
|
||||
|
||||
manifestModule =
|
||||
let
|
||||
mkTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
in
|
||||
lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = lib.literalExpression "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkYaml docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
mkYaml name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
enabledManifests = with builtins; filter (m: m.enable) (attrValues cfg.manifests);
|
||||
linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
|
||||
linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
|
||||
linkChartEntry =
|
||||
let
|
||||
mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
in
|
||||
name: value:
|
||||
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
|
||||
|
||||
activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
|
||||
${lib.optionalString (
|
||||
builtins.length enabledManifests > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
|
||||
${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
||||
${lib.optionalString (
|
||||
builtins.length cfg.images > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
|
||||
|
||||
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
|
||||
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
|
||||
${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
||||
@@ -127,11 +230,148 @@ in
|
||||
default = null;
|
||||
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
|
||||
};
|
||||
|
||||
manifests = mkOption {
|
||||
type = types.attrsOf manifestModule;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
};
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
'';
|
||||
description = ''
|
||||
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
||||
Note that deleting manifest files will not remove or otherwise modify the resources
|
||||
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
|
||||
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
||||
This option only makes sense on server nodes (`role = server`).
|
||||
Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
|
||||
for further information.
|
||||
'';
|
||||
};
|
||||
|
||||
charts = mkOption {
|
||||
type = with types; attrsOf (either path package);
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
nginx = ../charts/my-nginx-chart.tgz;
|
||||
redis = ../charts/my-redis-chart.tgz;
|
||||
'';
|
||||
description = ''
|
||||
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
||||
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
||||
The specified charts will only be placed on the file system and made available to the
|
||||
Kubernetes APIServer from within the cluster, you may use the
|
||||
[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy the charts. This option only makes sense on server nodes
|
||||
(`role = server`).
|
||||
'';
|
||||
};
|
||||
|
||||
images = mkOption {
|
||||
type = with types; listOf package;
|
||||
default = [ ];
|
||||
example = lib.literalExpression ''
|
||||
[
|
||||
(pkgs.dockerTools.pullImage {
|
||||
imageName = "docker.io/bitnami/keycloak";
|
||||
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
|
||||
sha256 = "0imblp0kw9vkcr7sp962jmj20fpmb3hvd3hmf4cs4x04klnq3k90";
|
||||
finalImageTag = "21.1.2-debian-11-r0";
|
||||
})
|
||||
]
|
||||
'';
|
||||
description = ''
|
||||
List of derivations that provide container images.
|
||||
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
|
||||
by the k3s agent. This option only makes sense on nodes with an enabled agent.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# implementation
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
warnings =
|
||||
(lib.optional (cfg.role != "server" && cfg.manifests != { })
|
||||
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
||||
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
|
||||
"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node"
|
||||
);
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != "");
|
||||
@@ -178,6 +418,7 @@ in
|
||||
LimitCORE = "infinity";
|
||||
TasksMax = "infinity";
|
||||
EnvironmentFile = cfg.environmentFile;
|
||||
ExecStartPre = activateK3sContent;
|
||||
ExecStart = concatStringsSep " \\\n " (
|
||||
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
|
||||
++ (optional cfg.clusterInit "--cluster-init")
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
k3s,
|
||||
...
|
||||
}:
|
||||
let
|
||||
pauseImageEnv = pkgs.buildEnv {
|
||||
name = "k3s-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
tini
|
||||
(hiPrio coreutils)
|
||||
busybox
|
||||
];
|
||||
};
|
||||
pauseImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/pause";
|
||||
tag = "local";
|
||||
copyToRoot = pauseImageEnv;
|
||||
config.Entrypoint = [
|
||||
"/bin/tini"
|
||||
"--"
|
||||
"/bin/sleep"
|
||||
"inf"
|
||||
];
|
||||
};
|
||||
helloImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/hello";
|
||||
tag = "local";
|
||||
copyToRoot = pkgs.hello;
|
||||
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${k3s.name}-auto-deploy";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = [ k3s ];
|
||||
|
||||
# k3s uses enough resources the default vm fails.
|
||||
virtualisation.memorySize = 1536;
|
||||
virtualisation.diskSize = 4096;
|
||||
|
||||
services.k3s.enable = true;
|
||||
services.k3s.role = "server";
|
||||
services.k3s.package = k3s;
|
||||
# Slightly reduce resource usage
|
||||
services.k3s.extraFlags = builtins.toString [
|
||||
"--disable coredns"
|
||||
"--disable local-storage"
|
||||
"--disable metrics-server"
|
||||
"--disable servicelb"
|
||||
"--disable traefik"
|
||||
"--pause-image test.local/pause:local"
|
||||
];
|
||||
services.k3s.images = [
|
||||
pauseImage
|
||||
helloImage
|
||||
];
|
||||
services.k3s.manifests = {
|
||||
absent = {
|
||||
enable = false;
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "absent";
|
||||
};
|
||||
};
|
||||
|
||||
present = {
|
||||
target = "foo-namespace.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "foo";
|
||||
};
|
||||
};
|
||||
|
||||
hello.content = {
|
||||
apiVersion = "batch/v1";
|
||||
kind = "Job";
|
||||
metadata.name = "hello";
|
||||
spec = {
|
||||
template.spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "hello";
|
||||
image = "test.local/hello:local";
|
||||
}
|
||||
];
|
||||
restartPolicy = "OnFailure";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
machine.wait_for_unit("k3s")
|
||||
# check existence of the manifest files
|
||||
machine.fail("ls /var/lib/rancher/k3s/server/manifests/absent.yaml")
|
||||
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/foo-namespace.yaml")
|
||||
machine.succeed("ls /var/lib/rancher/k3s/server/manifests/hello.yaml")
|
||||
|
||||
# check if container images got imported
|
||||
machine.succeed("crictl img | grep 'test\.local/pause'")
|
||||
machine.succeed("crictl img | grep 'test\.local/hello'")
|
||||
|
||||
# check if resources of manifests got created
|
||||
machine.wait_until_succeeds("kubectl get ns foo")
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello")
|
||||
machine.fail("kubectl get ns absent")
|
||||
|
||||
machine.shutdown()
|
||||
'';
|
||||
}
|
||||
)
|
||||
@@ -19,4 +19,6 @@ in
|
||||
single-node = lib.mapAttrs (_: k3s: import ./single-node.nix { inherit system pkgs k3s; }) allK3s;
|
||||
# Run a multi-node k3s cluster and verify pod networking works across nodes
|
||||
multi-node = lib.mapAttrs (_: k3s: import ./multi-node.nix { inherit system pkgs k3s; }) allK3s;
|
||||
# Test wether container images are imported and auto deploying manifests work
|
||||
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user