gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/k3s: factor out generic functionality

Browse Source 
Also slightly reworded a few option descriptions, commit prepares for merge with nixos/rke2.
This commit is contained in:
azey
2025-10-02 15:45:27 +02:00
parent 964c358544
commit 2ce16ee67e
2 changed files with 955 additions and 860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
nixos/modules/services/cluster/k3s/default.nix
View File

@@ -5,34 +5,28 @@
...
}:
let
cfg = config.services.k3s;
removeOption =
config: instruction:
lib.mkRemovedOptionModule (
[
"services"
"k3s"
]
++ config
) instruction;
mkRancherModule =
{
# name used in paths/names, e.g. k3s
name ? null,
# extra flags to pass to the binary before user-defined extraFlags
extraBinFlags ? [ ],
}:
let
cfg = config.services.${name};
manifestDir = "/var/lib/rancher/${name}/server/manifests";
imageDir = "/var/lib/rancher/${name}/agent/images";
containerdConfigTemplateFile = "/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl";
manifestDir = "/var/lib/rancher/k3s/server/manifests";
chartDir = "/var/lib/rancher/k3s/server/static/charts";
imageDir = "/var/lib/rancher/k3s/agent/images";
containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
yamlFormat = pkgs.formats.yaml { };
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
# Manifests need a valid YAML suffix to be respected by k3s
# Manifests need a valid YAML suffix to be respected
mkManifestTarget =
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
# Produces a list containing all duplicate manifest names
duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.manifests
);
# Produces a list containing all duplicate chart names
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.charts
);
# Converts YAML -> JSON -> Nix
fromYaml =
@@ -109,7 +103,7 @@ let
name: value:
let
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
# use JSON for values as it's a subset of YAML and understood by the rancher Helm controller
valuesContent = builtins.toJSON chartValues;
in
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
@@ -129,7 +123,7 @@ let
} value.extraFieldDefinitions;
# Generate a HelmChart custom resource together with extraDeploy manifests. This
# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
# generates possibly a multi document YAML file that the auto deploy mechanism
# deploys.
mkAutoDeployChartManifest = name: value: {
# target is the final name of the link created for the manifest file
@@ -360,8 +354,8 @@ let
target = lib.mkDefault (mkManifestTarget name);
source = lib.mkIf (config.content != null) (
let
name' = "k3s-manifest-" + builtins.baseNameOf name;
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
name' = "${name}-manifest-" + builtins.baseNameOf name;
docName = "${name}-manifest-doc-" + builtins.baseNameOf name;
mkSource =
value:
if builtins.isList value then
@@ -381,29 +375,17 @@ let
);
in
{
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
paths = { inherit manifestDir imageDir containerdConfigTemplateFile; };
# interface
options.services.k3s = {
enable = lib.mkEnableOption "k3s";
package = lib.mkPackageOption pkgs "k3s" { };
options = {
enable = lib.mkEnableOption name;
package = lib.mkPackageOption pkgs name { };
role = lib.mkOption {
description = ''
Whether k3s should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- Starts by default as a standalone server using an embedded sqlite datastore.
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
- Configure `serverAddr` to join an already-initialized HA cluster.
If it's an agent:
- `serverAddr` is required.
'';
description = "Whether ${name} should run as a server or agent.";
default = "server";
type = lib.types.enum [
"server"
@@ -413,45 +395,17 @@ in
serverAddr = lib.mkOption {
type = lib.types.str;
description = ''
The k3s server to connect to.
Servers and agents need to communicate each other. Read
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
to know how to configure the firewall.
'';
description = "The ${name} server to connect to, used to join a cluster.";
example = "https://10.0.0.10:6443";
default = "";
};
clusterInit = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Initialize HA cluster using an embedded etcd datastore.
If this option is `false` and `role` is `server`
On a server that was using the default embedded sqlite backend,
enabling this option will migrate to an embedded etcd DB.
If an HA cluster using the embedded etcd datastore was already initialized,
this option has no effect.
This option only makes sense in a server that is not connecting to another server.
If you are configuring an HA cluster with an embedded etcd,
the 1st server must have `clusterInit = true`
and other servers must connect to it using `serverAddr`.
'';
};
token = lib.mkOption {
type = lib.types.str;
description = ''
The k3s token to use when connecting to a server.
The ${name} token to use when connecting to a server.
WARNING: This option will expose store your token unencrypted world-readable in the nix store.
WARNING: This option will expose your token unencrypted in the world-readable nix store.
If this is undesired use the tokenFile option instead.
'';
default = "";
@@ -459,30 +413,24 @@ in
tokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "File path containing k3s token to use when connecting to the server.";
description = "File path containing ${name} token to use when connecting to the server.";
default = null;
};
extraFlags = lib.mkOption {
description = "Extra flags to pass to the k3s command.";
description = "Extra flags to pass to the ${name} command.";
type = with lib.types; either str (listOf str);
default = [ ];
example = [
"--disable traefik"
"--etcd-expose-metrics"
"--cluster-cidr 10.24.0.0/16"
];
};
disableAgent = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Only run the server. This option only makes sense for a server.";
};
environmentFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = ''
File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
File path containing environment variables for configuring the ${name} service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
'';
default = null;
};
@@ -490,7 +438,7 @@ in
configPath = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
description = "File path containing the ${name} YAML config. This is useful when the config is generated (for example on boot).";
};
manifests = lib.mkOption {
@@ -573,7 +521,7 @@ in
};
'';
description = ''
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
Auto-deploying manifests that are linked to {file}`${manifestDir}` before ${name} starts.
Note that deleting manifest files will not remove or otherwise modify the resources
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
@@ -583,28 +531,11 @@ in
'';
};
charts = lib.mkOption {
type = with lib.types; attrsOf (either path package);
default = { };
example = lib.literalExpression ''
nginx = ../charts/my-nginx-chart.tgz;
redis = ../charts/my-redis-chart.tgz;
'';
description = ''
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
The specified charts will only be placed on the file system and made available to the
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
'';
};
containerdConfigTemplate = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = lib.literalExpression ''
# Base K3s config
# Base config
{{ template "base" . }}
# Add a custom runtime
@@ -615,8 +546,8 @@ in
'';
description = ''
Config template for containerd, to be placed at
`/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
`/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl`.
See the docs on [configuring containerd](https://docs.${name}.io/advanced#configuring-containerd).
'';
};
@@ -631,16 +562,12 @@ in
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
config.services.k3s.package.airgap-images
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
use, if you want to pre-provision this node with all k3s container images. This option
only makes sense on nodes with an enabled agent.
All images are linked to {file}`${imageDir}` before ${name} starts and are consequently imported
by the ${name} agent. This option only makes sense on nodes with an enabled agent.
'';
};
@@ -696,14 +623,14 @@ in
default = { };
example = {
mode = "nftables";
clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
clientConnection.kubeconfig = "/var/lib/rancher/${name}/agent/kubeproxy.kubeconfig";
};
description = ''
Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
configuration that can be configured via a file is defined by the
[KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
struct. Note that the kubeconfig param will be overriden by `clientConnection.kubeconfig`, so you must
set the `clientConnection.kubeconfig` option if you want to use `extraKubeProxyConfig`.
'';
};
@@ -748,71 +675,43 @@ in
}
'';
description = ''
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
attribute names that are also used in the [](#opt-services.k3s.manifests) and
[](#opt-services.k3s.charts) options. Manifests with the same name will override
auto deploying charts with the same name. Similiarly, charts with the same name will
overwrite the Helm chart contained in auto deploying charts. This option only makes
sense on server nodes (`role = server`). See the
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
Auto deploying Helm charts that are installed by the ${name} Helm controller. Avoid using
attribute names that are also used in the [](#opt-services.${name}.manifests) option.
Manifests with the same name will override auto deploying charts with the same name.
This option only makes sense on server nodes (`role = server`). See the
[${name} Helm documentation](https://docs.${name}.io/helm) for further information.
'';
};
};
# implementation
config = lib.mkIf cfg.enable {
config = {
warnings =
(lib.optional (cfg.role != "server" && cfg.manifests != { })
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
"${name}: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
"${name}: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (duplicateManifests != [ ])
"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
"${name}: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
)
++ (lib.optional (duplicateCharts != [ ])
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
)
++ (lib.optional (
cfg.disableAgent && cfg.images != [ ]
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
++ (lib.optional (
cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
) "${name}: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
++ (lib.optional
(cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
"k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
"${name}: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
);
assertions = [
{
assertion = cfg.role == "agent" -> !cfg.disableAgent;
message = "k3s: disableAgent must be false if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !cfg.clusterInit;
message = "k3s: clusterInit must be false if role is 'agent'";
}
];
environment.systemPackages = [ config.services.${name}.package ];
environment.systemPackages = [ config.services.k3s.package ];
# Use systemd-tmpfiles to activate k3s content
systemd.tmpfiles.settings."10-k3s" =
# Use systemd-tmpfiles to activate content
systemd.tmpfiles.settings."10-${name}" =
let
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
# Merge charts with charts contained in enabled auto deploying charts
helmCharts =
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
))
// cfg.charts;
# Make a systemd-tmpfiles rule for a manifest
mkManifestRule = manifest: {
name = "${manifestDir}/${manifest.target}";
@@ -820,15 +719,6 @@ in
"L+".argument = "${manifest.source}";
};
};
# Ensure that all chart targets have a .tgz suffix
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
# Make a systemd-tmpfiles rule for a chart
mkChartRule = target: source: {
name = "${chartDir}/${mkChartTarget target}";
value = {
"L+".argument = "${source}";
};
};
# Make a systemd-tmpfiles rule for a container image
mkImageRule = image: {
name = "${imageDir}/${image.name}";
@@ -838,7 +728,6 @@ in
};
in
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
// (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
// (builtins.listToAttrs (map mkImageRule cfg.images))
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
${containerdConfigTemplateFile} = {
@@ -846,14 +735,14 @@ in
};
});
systemd.services.k3s =
systemd.services.${name} =
let
kubeletParams =
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
})
// cfg.extraKubeletConfig;
kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
kubeletConfig = (pkgs.formats.yaml { }).generate "${name}-kubelet-config" (
{
apiVersion = "kubelet.config.k8s.io/v1beta1";
kind = "KubeletConfiguration";
@@ -861,7 +750,7 @@ in
// kubeletParams
);
kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
kubeProxyConfig = (pkgs.formats.yaml { }).generate "${name}-kubeProxy-config" (
{
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
kind = "KubeProxyConfiguration";
@@ -870,7 +759,7 @@ in
);
in
{
description = "k3s service";
description = "${name} service";
after = [
"firewall.service"
"network-online.target"
@@ -894,20 +783,35 @@ in
TasksMax = "infinity";
EnvironmentFile = cfg.environmentFile;
ExecStart = lib.concatStringsSep " \\\n " (
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
++ (lib.optional cfg.clusterInit "--cluster-init")
++ (lib.optional cfg.disableAgent "--disable-agent")
[ "${cfg.package}/bin/${name} ${cfg.role}" ]
++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
++ extraBinFlags
++ (lib.flatten cfg.extraFlags)
);
};
};
};
meta.maintainers = lib.teams.k3s.members;
};
in
{
imports =
# pass mkRancherModule explicitly instead of via
# _modules.args to prevent infinite recursion
builtins.map (
f:
import f {
inherit config lib;
inherit mkRancherModule;
}
) [ ./k3s.nix ];
meta.maintainers =
with lib.maintainers;
[ azey7f ] # modules only
++ lib.teams.k3s.members;
}

191
nixos/modules/services/cluster/k3s/k3s.nix Normal file
View File

@@ -0,0 +1,191 @@
{
config,
lib,
mkRancherModule,
...
}:
let
cfg = config.services.k3s;
baseModule = mkRancherModule {
name = "k3s";
extraBinFlags =
(lib.optional cfg.clusterInit "--cluster-init")
++ (lib.optional cfg.disableAgent "--disable-agent");
};
removeOption =
config: instruction:
lib.mkRemovedOptionModule (
[
"services"
"k3s"
]
++ config
) instruction;
chartDir = "/var/lib/rancher/k3s/server/static/charts";
# Produces a list containing all duplicate chart names
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.charts
);
in
{
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
# interface
options.services.k3s = lib.recursiveUpdate baseModule.options {
# option overrides
role.description = ''
Whether k3s should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- Starts by default as a standalone server using an embedded sqlite datastore.
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
- Configure `serverAddr` to join an already-initialized HA cluster.
If it's an agent:
- `serverAddr` is required.
'';
serverAddr.description = ''
The k3s server to connect to.
Servers and agents need to communicate each other. Read
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
to know how to configure the firewall.
'';
images = {
example = lib.literalExpression ''
[
(pkgs.dockerTools.pullImage {
imageName = "docker.io/bitnami/keycloak";
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
config.services.k3s.package.airgap-images
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${baseModule.imageDir}` before k3s starts and are consequently imported
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
use, if you want to pre-provision this node with all k3s container images. This option
only makes sense on nodes with an enabled agent.
'';
};
autoDeployCharts.description = ''
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid using
attribute names that are also used in the [](#opt-services.k3s.manifests) and
[](#opt-services.k3s.charts) options. Manifests with the same name will override
auto deploying charts with the same name. Similiarly, charts with the same name will
overwrite the Helm chart contained in auto deploying charts. This option only makes
sense on server nodes (`role = server`). See the
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
'';
# k3s-specific options
clusterInit = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Initialize HA cluster using an embedded etcd datastore.
If this option is `false` and `role` is `server`
On a server that was using the default embedded sqlite backend,
enabling this option will migrate to an embedded etcd DB.
If an HA cluster using the embedded etcd datastore was already initialized,
this option has no effect.
This option only makes sense in a server that is not connecting to another server.
If you are configuring an HA cluster with an embedded etcd,
the 1st server must have `clusterInit = true`
and other servers must connect to it using `serverAddr`.
'';
};
disableAgent = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Only run the server. This option only makes sense for a server.";
};
charts = lib.mkOption {
type = with lib.types; attrsOf (either path package);
default = { };
example = lib.literalExpression ''
nginx = ../charts/my-nginx-chart.tgz;
redis = ../charts/my-redis-chart.tgz;
'';
description = ''
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
The specified charts will only be placed on the file system and made available to the
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
'';
};
};
# implementation
config = lib.mkIf cfg.enable (
lib.recursiveUpdate baseModule.config {
warnings =
(lib.optional (cfg.role != "server" && cfg.charts != { })
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (duplicateCharts != [ ])
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
)
++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node."
);
assertions = [
{
assertion = cfg.role == "agent" -> !cfg.disableAgent;
message = "k3s: disableAgent must be false if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !cfg.clusterInit;
message = "k3s: clusterInit must be false if role is 'agent'";
}
];
systemd.tmpfiles.settings."10-k3s" =
let
# Merge charts with charts contained in enabled auto deploying charts
helmCharts =
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
))
// cfg.charts;
# Ensure that all chart targets have a .tgz suffix
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
# Make a systemd-tmpfiles rule for a chart
mkChartRule = target: source: {
name = "${chartDir}/${mkChartTarget target}";
value = {
"L+".argument = "${source}";
};
};
in
lib.mapAttrs' (n: v: mkChartRule n v) helmCharts;
}
);
}