nixos/k3s: factor out generic functionality
Also slightly reworded a few option descriptions, commit prepares for merge with nixos/rke2.
This commit is contained in:
@@ -5,34 +5,28 @@
|
|||||||
...
|
...
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
cfg = config.services.k3s;
|
mkRancherModule =
|
||||||
removeOption =
|
{
|
||||||
config: instruction:
|
# name used in paths/names, e.g. k3s
|
||||||
lib.mkRemovedOptionModule (
|
name ? null,
|
||||||
[
|
# extra flags to pass to the binary before user-defined extraFlags
|
||||||
"services"
|
extraBinFlags ? [ ],
|
||||||
"k3s"
|
}:
|
||||||
]
|
let
|
||||||
++ config
|
cfg = config.services.${name};
|
||||||
) instruction;
|
manifestDir = "/var/lib/rancher/${name}/server/manifests";
|
||||||
|
imageDir = "/var/lib/rancher/${name}/agent/images";
|
||||||
|
containerdConfigTemplateFile = "/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl";
|
||||||
|
|
||||||
manifestDir = "/var/lib/rancher/k3s/server/manifests";
|
|
||||||
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
|
||||||
imageDir = "/var/lib/rancher/k3s/agent/images";
|
|
||||||
containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
|
|
||||||
yamlFormat = pkgs.formats.yaml { };
|
yamlFormat = pkgs.formats.yaml { };
|
||||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||||
# Manifests need a valid YAML suffix to be respected by k3s
|
# Manifests need a valid YAML suffix to be respected
|
||||||
mkManifestTarget =
|
mkManifestTarget =
|
||||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||||
# Produces a list containing all duplicate manifest names
|
# Produces a list containing all duplicate manifest names
|
||||||
duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
|
duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
|
||||||
builtins.attrNames cfg.manifests
|
builtins.attrNames cfg.manifests
|
||||||
);
|
);
|
||||||
# Produces a list containing all duplicate chart names
|
|
||||||
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
|
|
||||||
builtins.attrNames cfg.charts
|
|
||||||
);
|
|
||||||
|
|
||||||
# Converts YAML -> JSON -> Nix
|
# Converts YAML -> JSON -> Nix
|
||||||
fromYaml =
|
fromYaml =
|
||||||
@@ -109,7 +103,7 @@ let
|
|||||||
name: value:
|
name: value:
|
||||||
let
|
let
|
||||||
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
|
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
|
||||||
# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
|
# use JSON for values as it's a subset of YAML and understood by the rancher Helm controller
|
||||||
valuesContent = builtins.toJSON chartValues;
|
valuesContent = builtins.toJSON chartValues;
|
||||||
in
|
in
|
||||||
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
|
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
|
||||||
@@ -129,7 +123,7 @@ let
|
|||||||
} value.extraFieldDefinitions;
|
} value.extraFieldDefinitions;
|
||||||
|
|
||||||
# Generate a HelmChart custom resource together with extraDeploy manifests. This
|
# Generate a HelmChart custom resource together with extraDeploy manifests. This
|
||||||
# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
|
# generates possibly a multi document YAML file that the auto deploy mechanism
|
||||||
# deploys.
|
# deploys.
|
||||||
mkAutoDeployChartManifest = name: value: {
|
mkAutoDeployChartManifest = name: value: {
|
||||||
# target is the final name of the link created for the manifest file
|
# target is the final name of the link created for the manifest file
|
||||||
@@ -360,8 +354,8 @@ let
|
|||||||
target = lib.mkDefault (mkManifestTarget name);
|
target = lib.mkDefault (mkManifestTarget name);
|
||||||
source = lib.mkIf (config.content != null) (
|
source = lib.mkIf (config.content != null) (
|
||||||
let
|
let
|
||||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
name' = "${name}-manifest-" + builtins.baseNameOf name;
|
||||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
docName = "${name}-manifest-doc-" + builtins.baseNameOf name;
|
||||||
mkSource =
|
mkSource =
|
||||||
value:
|
value:
|
||||||
if builtins.isList value then
|
if builtins.isList value then
|
||||||
@@ -379,31 +373,19 @@ let
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
paths = { inherit manifestDir imageDir containerdConfigTemplateFile; };
|
||||||
|
|
||||||
# interface
|
# interface
|
||||||
options.services.k3s = {
|
|
||||||
enable = lib.mkEnableOption "k3s";
|
|
||||||
|
|
||||||
package = lib.mkPackageOption pkgs "k3s" { };
|
options = {
|
||||||
|
enable = lib.mkEnableOption name;
|
||||||
|
|
||||||
|
package = lib.mkPackageOption pkgs name { };
|
||||||
|
|
||||||
role = lib.mkOption {
|
role = lib.mkOption {
|
||||||
description = ''
|
description = "Whether ${name} should run as a server or agent.";
|
||||||
Whether k3s should run as a server or agent.
|
|
||||||
|
|
||||||
If it's a server:
|
|
||||||
|
|
||||||
- By default it also runs workloads as an agent.
|
|
||||||
- Starts by default as a standalone server using an embedded sqlite datastore.
|
|
||||||
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
|
|
||||||
- Configure `serverAddr` to join an already-initialized HA cluster.
|
|
||||||
|
|
||||||
If it's an agent:
|
|
||||||
|
|
||||||
- `serverAddr` is required.
|
|
||||||
'';
|
|
||||||
default = "server";
|
default = "server";
|
||||||
type = lib.types.enum [
|
type = lib.types.enum [
|
||||||
"server"
|
"server"
|
||||||
@@ -413,45 +395,17 @@ in
|
|||||||
|
|
||||||
serverAddr = lib.mkOption {
|
serverAddr = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
description = ''
|
description = "The ${name} server to connect to, used to join a cluster.";
|
||||||
The k3s server to connect to.
|
|
||||||
|
|
||||||
Servers and agents need to communicate each other. Read
|
|
||||||
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
|
|
||||||
to know how to configure the firewall.
|
|
||||||
'';
|
|
||||||
example = "https://10.0.0.10:6443";
|
example = "https://10.0.0.10:6443";
|
||||||
default = "";
|
default = "";
|
||||||
};
|
};
|
||||||
|
|
||||||
clusterInit = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
description = ''
|
|
||||||
Initialize HA cluster using an embedded etcd datastore.
|
|
||||||
|
|
||||||
If this option is `false` and `role` is `server`
|
|
||||||
|
|
||||||
On a server that was using the default embedded sqlite backend,
|
|
||||||
enabling this option will migrate to an embedded etcd DB.
|
|
||||||
|
|
||||||
If an HA cluster using the embedded etcd datastore was already initialized,
|
|
||||||
this option has no effect.
|
|
||||||
|
|
||||||
This option only makes sense in a server that is not connecting to another server.
|
|
||||||
|
|
||||||
If you are configuring an HA cluster with an embedded etcd,
|
|
||||||
the 1st server must have `clusterInit = true`
|
|
||||||
and other servers must connect to it using `serverAddr`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
token = lib.mkOption {
|
token = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
description = ''
|
description = ''
|
||||||
The k3s token to use when connecting to a server.
|
The ${name} token to use when connecting to a server.
|
||||||
|
|
||||||
WARNING: This option will expose store your token unencrypted world-readable in the nix store.
|
WARNING: This option will expose your token unencrypted in the world-readable nix store.
|
||||||
If this is undesired use the tokenFile option instead.
|
If this is undesired use the tokenFile option instead.
|
||||||
'';
|
'';
|
||||||
default = "";
|
default = "";
|
||||||
@@ -459,30 +413,24 @@ in
|
|||||||
|
|
||||||
tokenFile = lib.mkOption {
|
tokenFile = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.path;
|
type = lib.types.nullOr lib.types.path;
|
||||||
description = "File path containing k3s token to use when connecting to the server.";
|
description = "File path containing ${name} token to use when connecting to the server.";
|
||||||
default = null;
|
default = null;
|
||||||
};
|
};
|
||||||
|
|
||||||
extraFlags = lib.mkOption {
|
extraFlags = lib.mkOption {
|
||||||
description = "Extra flags to pass to the k3s command.";
|
description = "Extra flags to pass to the ${name} command.";
|
||||||
type = with lib.types; either str (listOf str);
|
type = with lib.types; either str (listOf str);
|
||||||
default = [ ];
|
default = [ ];
|
||||||
example = [
|
example = [
|
||||||
"--disable traefik"
|
"--etcd-expose-metrics"
|
||||||
"--cluster-cidr 10.24.0.0/16"
|
"--cluster-cidr 10.24.0.0/16"
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
disableAgent = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
description = "Only run the server. This option only makes sense for a server.";
|
|
||||||
};
|
|
||||||
|
|
||||||
environmentFile = lib.mkOption {
|
environmentFile = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.path;
|
type = lib.types.nullOr lib.types.path;
|
||||||
description = ''
|
description = ''
|
||||||
File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
|
File path containing environment variables for configuring the ${name} service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
|
||||||
'';
|
'';
|
||||||
default = null;
|
default = null;
|
||||||
};
|
};
|
||||||
@@ -490,7 +438,7 @@ in
|
|||||||
configPath = lib.mkOption {
|
configPath = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.path;
|
type = lib.types.nullOr lib.types.path;
|
||||||
default = null;
|
default = null;
|
||||||
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
|
description = "File path containing the ${name} YAML config. This is useful when the config is generated (for example on boot).";
|
||||||
};
|
};
|
||||||
|
|
||||||
manifests = lib.mkOption {
|
manifests = lib.mkOption {
|
||||||
@@ -573,7 +521,7 @@ in
|
|||||||
};
|
};
|
||||||
'';
|
'';
|
||||||
description = ''
|
description = ''
|
||||||
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
Auto-deploying manifests that are linked to {file}`${manifestDir}` before ${name} starts.
|
||||||
Note that deleting manifest files will not remove or otherwise modify the resources
|
Note that deleting manifest files will not remove or otherwise modify the resources
|
||||||
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
|
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
|
||||||
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
||||||
@@ -583,28 +531,11 @@ in
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
charts = lib.mkOption {
|
|
||||||
type = with lib.types; attrsOf (either path package);
|
|
||||||
default = { };
|
|
||||||
example = lib.literalExpression ''
|
|
||||||
nginx = ../charts/my-nginx-chart.tgz;
|
|
||||||
redis = ../charts/my-redis-chart.tgz;
|
|
||||||
'';
|
|
||||||
description = ''
|
|
||||||
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
|
||||||
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
|
||||||
The specified charts will only be placed on the file system and made available to the
|
|
||||||
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
|
|
||||||
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
|
|
||||||
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
containerdConfigTemplate = lib.mkOption {
|
containerdConfigTemplate = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
default = null;
|
default = null;
|
||||||
example = lib.literalExpression ''
|
example = lib.literalExpression ''
|
||||||
# Base K3s config
|
# Base config
|
||||||
{{ template "base" . }}
|
{{ template "base" . }}
|
||||||
|
|
||||||
# Add a custom runtime
|
# Add a custom runtime
|
||||||
@@ -615,8 +546,8 @@ in
|
|||||||
'';
|
'';
|
||||||
description = ''
|
description = ''
|
||||||
Config template for containerd, to be placed at
|
Config template for containerd, to be placed at
|
||||||
`/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
|
`/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl`.
|
||||||
See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
|
See the docs on [configuring containerd](https://docs.${name}.io/advanced#configuring-containerd).
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -631,16 +562,12 @@ in
|
|||||||
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
|
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
|
||||||
finalImageTag = "21.1.2-debian-11-r0";
|
finalImageTag = "21.1.2-debian-11-r0";
|
||||||
})
|
})
|
||||||
|
|
||||||
config.services.k3s.package.airgap-images
|
|
||||||
]
|
]
|
||||||
'';
|
'';
|
||||||
description = ''
|
description = ''
|
||||||
List of derivations that provide container images.
|
List of derivations that provide container images.
|
||||||
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
|
All images are linked to {file}`${imageDir}` before ${name} starts and are consequently imported
|
||||||
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
|
by the ${name} agent. This option only makes sense on nodes with an enabled agent.
|
||||||
use, if you want to pre-provision this node with all k3s container images. This option
|
|
||||||
only makes sense on nodes with an enabled agent.
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -696,14 +623,14 @@ in
|
|||||||
default = { };
|
default = { };
|
||||||
example = {
|
example = {
|
||||||
mode = "nftables";
|
mode = "nftables";
|
||||||
clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
|
clientConnection.kubeconfig = "/var/lib/rancher/${name}/agent/kubeproxy.kubeconfig";
|
||||||
};
|
};
|
||||||
description = ''
|
description = ''
|
||||||
Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
|
Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
|
||||||
configuration that can be configured via a file is defined by the
|
configuration that can be configured via a file is defined by the
|
||||||
[KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
|
[KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
|
||||||
struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
|
struct. Note that the kubeconfig param will be overriden by `clientConnection.kubeconfig`, so you must
|
||||||
set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
|
set the `clientConnection.kubeconfig` option if you want to use `extraKubeProxyConfig`.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -748,71 +675,43 @@ in
|
|||||||
}
|
}
|
||||||
'';
|
'';
|
||||||
description = ''
|
description = ''
|
||||||
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
|
Auto deploying Helm charts that are installed by the ${name} Helm controller. Avoid using
|
||||||
attribute names that are also used in the [](#opt-services.k3s.manifests) and
|
attribute names that are also used in the [](#opt-services.${name}.manifests) option.
|
||||||
[](#opt-services.k3s.charts) options. Manifests with the same name will override
|
Manifests with the same name will override auto deploying charts with the same name.
|
||||||
auto deploying charts with the same name. Similiarly, charts with the same name will
|
This option only makes sense on server nodes (`role = server`). See the
|
||||||
overwrite the Helm chart contained in auto deploying charts. This option only makes
|
[${name} Helm documentation](https://docs.${name}.io/helm) for further information.
|
||||||
sense on server nodes (`role = server`). See the
|
|
||||||
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# implementation
|
# implementation
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
config = {
|
||||||
warnings =
|
warnings =
|
||||||
(lib.optional (cfg.role != "server" && cfg.manifests != { })
|
(lib.optional (cfg.role != "server" && cfg.manifests != { })
|
||||||
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
|
"${name}: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
|
||||||
)
|
|
||||||
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
|
||||||
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
|
||||||
)
|
)
|
||||||
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
|
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
|
||||||
"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
|
"${name}: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
|
||||||
)
|
)
|
||||||
++ (lib.optional (duplicateManifests != [ ])
|
++ (lib.optional (duplicateManifests != [ ])
|
||||||
"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
|
"${name}: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
|
||||||
)
|
)
|
||||||
++ (lib.optional (duplicateCharts != [ ])
|
|
||||||
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
|
|
||||||
)
|
|
||||||
++ (lib.optional (
|
|
||||||
cfg.disableAgent && cfg.images != [ ]
|
|
||||||
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
|
|
||||||
++ (lib.optional (
|
++ (lib.optional (
|
||||||
cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
|
cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
|
||||||
) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
|
) "${name}: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
|
||||||
++ (lib.optional
|
++ (lib.optional
|
||||||
(cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
|
(cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
|
||||||
"k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
|
"${name}: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
|
||||||
);
|
);
|
||||||
|
|
||||||
assertions = [
|
environment.systemPackages = [ config.services.${name}.package ];
|
||||||
{
|
|
||||||
assertion = cfg.role == "agent" -> !cfg.disableAgent;
|
|
||||||
message = "k3s: disableAgent must be false if role is 'agent'";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
assertion = cfg.role == "agent" -> !cfg.clusterInit;
|
|
||||||
message = "k3s: clusterInit must be false if role is 'agent'";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
environment.systemPackages = [ config.services.k3s.package ];
|
# Use systemd-tmpfiles to activate content
|
||||||
|
systemd.tmpfiles.settings."10-${name}" =
|
||||||
# Use systemd-tmpfiles to activate k3s content
|
|
||||||
systemd.tmpfiles.settings."10-k3s" =
|
|
||||||
let
|
let
|
||||||
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
|
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
|
||||||
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
|
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
|
||||||
# Merge charts with charts contained in enabled auto deploying charts
|
|
||||||
helmCharts =
|
|
||||||
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
|
|
||||||
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
|
|
||||||
))
|
|
||||||
// cfg.charts;
|
|
||||||
# Make a systemd-tmpfiles rule for a manifest
|
# Make a systemd-tmpfiles rule for a manifest
|
||||||
mkManifestRule = manifest: {
|
mkManifestRule = manifest: {
|
||||||
name = "${manifestDir}/${manifest.target}";
|
name = "${manifestDir}/${manifest.target}";
|
||||||
@@ -820,15 +719,6 @@ in
|
|||||||
"L+".argument = "${manifest.source}";
|
"L+".argument = "${manifest.source}";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
# Ensure that all chart targets have a .tgz suffix
|
|
||||||
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
|
||||||
# Make a systemd-tmpfiles rule for a chart
|
|
||||||
mkChartRule = target: source: {
|
|
||||||
name = "${chartDir}/${mkChartTarget target}";
|
|
||||||
value = {
|
|
||||||
"L+".argument = "${source}";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
# Make a systemd-tmpfiles rule for a container image
|
# Make a systemd-tmpfiles rule for a container image
|
||||||
mkImageRule = image: {
|
mkImageRule = image: {
|
||||||
name = "${imageDir}/${image.name}";
|
name = "${imageDir}/${image.name}";
|
||||||
@@ -838,7 +728,6 @@ in
|
|||||||
};
|
};
|
||||||
in
|
in
|
||||||
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
|
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
|
||||||
// (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
|
|
||||||
// (builtins.listToAttrs (map mkImageRule cfg.images))
|
// (builtins.listToAttrs (map mkImageRule cfg.images))
|
||||||
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
|
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
|
||||||
${containerdConfigTemplateFile} = {
|
${containerdConfigTemplateFile} = {
|
||||||
@@ -846,14 +735,14 @@ in
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
systemd.services.k3s =
|
systemd.services.${name} =
|
||||||
let
|
let
|
||||||
kubeletParams =
|
kubeletParams =
|
||||||
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
|
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
|
||||||
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
|
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
|
||||||
})
|
})
|
||||||
// cfg.extraKubeletConfig;
|
// cfg.extraKubeletConfig;
|
||||||
kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
|
kubeletConfig = (pkgs.formats.yaml { }).generate "${name}-kubelet-config" (
|
||||||
{
|
{
|
||||||
apiVersion = "kubelet.config.k8s.io/v1beta1";
|
apiVersion = "kubelet.config.k8s.io/v1beta1";
|
||||||
kind = "KubeletConfiguration";
|
kind = "KubeletConfiguration";
|
||||||
@@ -861,7 +750,7 @@ in
|
|||||||
// kubeletParams
|
// kubeletParams
|
||||||
);
|
);
|
||||||
|
|
||||||
kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
|
kubeProxyConfig = (pkgs.formats.yaml { }).generate "${name}-kubeProxy-config" (
|
||||||
{
|
{
|
||||||
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
|
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
|
||||||
kind = "KubeProxyConfiguration";
|
kind = "KubeProxyConfiguration";
|
||||||
@@ -870,7 +759,7 @@ in
|
|||||||
);
|
);
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
description = "k3s service";
|
description = "${name} service";
|
||||||
after = [
|
after = [
|
||||||
"firewall.service"
|
"firewall.service"
|
||||||
"network-online.target"
|
"network-online.target"
|
||||||
@@ -894,20 +783,35 @@ in
|
|||||||
TasksMax = "infinity";
|
TasksMax = "infinity";
|
||||||
EnvironmentFile = cfg.environmentFile;
|
EnvironmentFile = cfg.environmentFile;
|
||||||
ExecStart = lib.concatStringsSep " \\\n " (
|
ExecStart = lib.concatStringsSep " \\\n " (
|
||||||
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
|
[ "${cfg.package}/bin/${name} ${cfg.role}" ]
|
||||||
++ (lib.optional cfg.clusterInit "--cluster-init")
|
|
||||||
++ (lib.optional cfg.disableAgent "--disable-agent")
|
|
||||||
++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
|
++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
|
||||||
++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
|
++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
|
||||||
++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
|
++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
|
||||||
++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
|
++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
|
||||||
++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
|
++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
|
||||||
++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
|
++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
|
||||||
|
++ extraBinFlags
|
||||||
++ (lib.flatten cfg.extraFlags)
|
++ (lib.flatten cfg.extraFlags)
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
imports =
|
||||||
|
# pass mkRancherModule explicitly instead of via
|
||||||
|
# _modules.args to prevent infinite recursion
|
||||||
|
builtins.map (
|
||||||
|
f:
|
||||||
|
import f {
|
||||||
|
inherit config lib;
|
||||||
|
inherit mkRancherModule;
|
||||||
|
}
|
||||||
|
) [ ./k3s.nix ];
|
||||||
|
|
||||||
meta.maintainers = lib.teams.k3s.members;
|
meta.maintainers =
|
||||||
|
with lib.maintainers;
|
||||||
|
[ azey7f ] # modules only
|
||||||
|
++ lib.teams.k3s.members;
|
||||||
}
|
}
|
||||||
|
|||||||
191
nixos/modules/services/cluster/k3s/k3s.nix
Normal file
191
nixos/modules/services/cluster/k3s/k3s.nix
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
{
|
||||||
|
config,
|
||||||
|
lib,
|
||||||
|
mkRancherModule,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
cfg = config.services.k3s;
|
||||||
|
baseModule = mkRancherModule {
|
||||||
|
name = "k3s";
|
||||||
|
extraBinFlags =
|
||||||
|
(lib.optional cfg.clusterInit "--cluster-init")
|
||||||
|
++ (lib.optional cfg.disableAgent "--disable-agent");
|
||||||
|
};
|
||||||
|
|
||||||
|
removeOption =
|
||||||
|
config: instruction:
|
||||||
|
lib.mkRemovedOptionModule (
|
||||||
|
[
|
||||||
|
"services"
|
||||||
|
"k3s"
|
||||||
|
]
|
||||||
|
++ config
|
||||||
|
) instruction;
|
||||||
|
|
||||||
|
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
||||||
|
# Produces a list containing all duplicate chart names
|
||||||
|
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
|
||||||
|
builtins.attrNames cfg.charts
|
||||||
|
);
|
||||||
|
in
|
||||||
|
{
|
||||||
|
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
||||||
|
|
||||||
|
# interface
|
||||||
|
|
||||||
|
options.services.k3s = lib.recursiveUpdate baseModule.options {
|
||||||
|
|
||||||
|
# option overrides
|
||||||
|
|
||||||
|
role.description = ''
|
||||||
|
Whether k3s should run as a server or agent.
|
||||||
|
|
||||||
|
If it's a server:
|
||||||
|
|
||||||
|
- By default it also runs workloads as an agent.
|
||||||
|
- Starts by default as a standalone server using an embedded sqlite datastore.
|
||||||
|
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
|
||||||
|
- Configure `serverAddr` to join an already-initialized HA cluster.
|
||||||
|
|
||||||
|
If it's an agent:
|
||||||
|
|
||||||
|
- `serverAddr` is required.
|
||||||
|
'';
|
||||||
|
|
||||||
|
serverAddr.description = ''
|
||||||
|
The k3s server to connect to.
|
||||||
|
|
||||||
|
Servers and agents need to communicate each other. Read
|
||||||
|
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
|
||||||
|
to know how to configure the firewall.
|
||||||
|
'';
|
||||||
|
|
||||||
|
images = {
|
||||||
|
example = lib.literalExpression ''
|
||||||
|
[
|
||||||
|
(pkgs.dockerTools.pullImage {
|
||||||
|
imageName = "docker.io/bitnami/keycloak";
|
||||||
|
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
|
||||||
|
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
|
||||||
|
finalImageTag = "21.1.2-debian-11-r0";
|
||||||
|
})
|
||||||
|
|
||||||
|
config.services.k3s.package.airgap-images
|
||||||
|
]
|
||||||
|
'';
|
||||||
|
description = ''
|
||||||
|
List of derivations that provide container images.
|
||||||
|
All images are linked to {file}`${baseModule.imageDir}` before k3s starts and are consequently imported
|
||||||
|
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
|
||||||
|
use, if you want to pre-provision this node with all k3s container images. This option
|
||||||
|
only makes sense on nodes with an enabled agent.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
autoDeployCharts.description = ''
|
||||||
|
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid using
|
||||||
|
attribute names that are also used in the [](#opt-services.k3s.manifests) and
|
||||||
|
[](#opt-services.k3s.charts) options. Manifests with the same name will override
|
||||||
|
auto deploying charts with the same name. Similiarly, charts with the same name will
|
||||||
|
overwrite the Helm chart contained in auto deploying charts. This option only makes
|
||||||
|
sense on server nodes (`role = server`). See the
|
||||||
|
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
|
||||||
|
'';
|
||||||
|
|
||||||
|
# k3s-specific options
|
||||||
|
|
||||||
|
clusterInit = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = ''
|
||||||
|
Initialize HA cluster using an embedded etcd datastore.
|
||||||
|
|
||||||
|
If this option is `false` and `role` is `server`
|
||||||
|
|
||||||
|
On a server that was using the default embedded sqlite backend,
|
||||||
|
enabling this option will migrate to an embedded etcd DB.
|
||||||
|
|
||||||
|
If an HA cluster using the embedded etcd datastore was already initialized,
|
||||||
|
this option has no effect.
|
||||||
|
|
||||||
|
This option only makes sense in a server that is not connecting to another server.
|
||||||
|
|
||||||
|
If you are configuring an HA cluster with an embedded etcd,
|
||||||
|
the 1st server must have `clusterInit = true`
|
||||||
|
and other servers must connect to it using `serverAddr`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
disableAgent = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Only run the server. This option only makes sense for a server.";
|
||||||
|
};
|
||||||
|
|
||||||
|
charts = lib.mkOption {
|
||||||
|
type = with lib.types; attrsOf (either path package);
|
||||||
|
default = { };
|
||||||
|
example = lib.literalExpression ''
|
||||||
|
nginx = ../charts/my-nginx-chart.tgz;
|
||||||
|
redis = ../charts/my-redis-chart.tgz;
|
||||||
|
'';
|
||||||
|
description = ''
|
||||||
|
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
||||||
|
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
||||||
|
The specified charts will only be placed on the file system and made available to the
|
||||||
|
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
|
||||||
|
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||||
|
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# implementation
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable (
|
||||||
|
lib.recursiveUpdate baseModule.config {
|
||||||
|
warnings =
|
||||||
|
(lib.optional (cfg.role != "server" && cfg.charts != { })
|
||||||
|
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
||||||
|
)
|
||||||
|
++ (lib.optional (duplicateCharts != [ ])
|
||||||
|
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
|
||||||
|
)
|
||||||
|
++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
|
||||||
|
"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node."
|
||||||
|
);
|
||||||
|
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = cfg.role == "agent" -> !cfg.disableAgent;
|
||||||
|
message = "k3s: disableAgent must be false if role is 'agent'";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
assertion = cfg.role == "agent" -> !cfg.clusterInit;
|
||||||
|
message = "k3s: clusterInit must be false if role is 'agent'";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
systemd.tmpfiles.settings."10-k3s" =
|
||||||
|
let
|
||||||
|
# Merge charts with charts contained in enabled auto deploying charts
|
||||||
|
helmCharts =
|
||||||
|
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
|
||||||
|
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
|
||||||
|
))
|
||||||
|
// cfg.charts;
|
||||||
|
# Ensure that all chart targets have a .tgz suffix
|
||||||
|
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||||
|
# Make a systemd-tmpfiles rule for a chart
|
||||||
|
mkChartRule = target: source: {
|
||||||
|
name = "${chartDir}/${mkChartTarget target}";
|
||||||
|
value = {
|
||||||
|
"L+".argument = "${source}";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
lib.mapAttrs' (n: v: mkChartRule n v) helmCharts;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user