gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/rke2: merge code with nixos/k3s (#447847)

Browse Source
This commit is contained in:
Ulrik Strid
2025-11-17 12:00:34 +00:00
committed by GitHub
parent bb43ec5ac3 cc777a7bda
commit 69ff903493
13 changed files with 1398 additions and 1364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.github/labeler.yml vendored
View File

@@ -261,7 +261,8 @@
- any:
- changed-files:
- any-glob-to-any-file:
- nixos/modules/services/cluster/k3s/**/*
- nixos/modules/services/cluster/rancher/default.nix
- nixos/modules/services/cluster/rancher/k3s.nix
- nixos/tests/k3s/**/*
- pkgs/applications/networking/cluster/k3s/**/*

3
nixos/doc/manual/release-notes/rl-2511.section.md
View File

@@ -476,6 +476,9 @@ and [release notes for v18](https://goteleport.com/docs/changelog/#1800-070325).
- `services.matter-server` now hosts a debug dashboard on the configured port. Open the port on the firewall with `services.matter-server.openFirewall`.
- `services.k3s` now shares most of its code with `services.rke2`. The merge resulted in both modules providing more options, with `services.rke2` receiving the most improvements.
Existing configurations for either module should not be affected.
- The new option [networking.ipips](#opt-networking.ipips) has been added to create IP within IP kind of tunnels (including 4in6, ip6ip6 and ipip).
With the existing [networking.sits](#opt-networking.sits) option (6in4), it is now possible to create all combinations of IPv4 and IPv6 encapsulation.

3
nixos/modules/module-list.nix
View File

@@ -474,7 +474,6 @@
./services/cluster/corosync/default.nix
./services/cluster/druid/default.nix
./services/cluster/hadoop/default.nix
./services/cluster/k3s/default.nix
./services/cluster/kubernetes/addon-manager.nix
./services/cluster/kubernetes/addons/dns.nix
./services/cluster/kubernetes/apiserver.nix
@@ -487,7 +486,7 @@
./services/cluster/kubernetes/scheduler.nix
./services/cluster/pacemaker/default.nix
./services/cluster/patroni/default.nix
./services/cluster/rke2/default.nix
./services/cluster/rancher/default.nix
./services/cluster/spark/default.nix
./services/cluster/temporal/default.nix
./services/computing/boinc/client.nix

913
nixos/modules/services/cluster/k3s/default.nix
View File

@@ -1,913 +0,0 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.k3s;
removeOption =
config: instruction:
lib.mkRemovedOptionModule (
[
"services"
"k3s"
]
++ config
) instruction;
manifestDir = "/var/lib/rancher/k3s/server/manifests";
chartDir = "/var/lib/rancher/k3s/server/static/charts";
imageDir = "/var/lib/rancher/k3s/agent/images";
containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
yamlFormat = pkgs.formats.yaml { };
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
# Manifests need a valid YAML suffix to be respected by k3s
mkManifestTarget =
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
# Produces a list containing all duplicate manifest names
duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.manifests
);
# Produces a list containing all duplicate chart names
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.charts
);
# Converts YAML -> JSON -> Nix
fromYaml =
path:
builtins.fromJSON (
builtins.readFile (
pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ pkgs.yq-go ]; } ''
yq --no-colors --output-format json ${path} > $out
''
)
);
# Replace prefixes and characters that are problematic in file names
cleanHelmChartName =
name:
let
woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
in
lib.replaceStrings
[
"/"
":"
]
[
"-"
"-"
]
woPrefix;
# Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
fetchHelm =
{
name,
repo,
version,
hash ? lib.fakeHash,
}:
let
isOci = lib.hasPrefix "oci://" repo;
pullCmd = if isOci then repo else "--repo ${repo} ${name}";
name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
in
pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
{
inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
nativeBuildInputs = with pkgs; [
kubernetes-helm
cacert
# Helm requires HOME to refer to a writable dir
writableTmpDirAsHomeHook
];
}
''
helm pull ${pullCmd} --version ${version}
mv ./*.tgz $out
'';
# Returns the path to a YAML manifest file
mkExtraDeployManifest =
x:
# x is a derivation that provides a YAML file
if lib.isDerivation x then
x.outPath
# x is an attribute set that needs to be converted to a YAML file
else if builtins.isAttrs x then
(yamlFormat.generate "extra-deploy-chart-manifest" x)
# assume x is a path to a YAML file
else
x;
# Generate a HelmChart custom resource.
mkHelmChartCR =
name: value:
let
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
valuesContent = builtins.toJSON chartValues;
in
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
# values
lib.recursiveUpdate {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChart";
metadata = {
inherit name;
namespace = "kube-system";
};
spec = {
inherit valuesContent;
inherit (value) targetNamespace createNamespace;
chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
};
} value.extraFieldDefinitions;
# Generate a HelmChart custom resource together with extraDeploy manifests. This
# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
# deploys.
mkAutoDeployChartManifest = name: value: {
# target is the final name of the link created for the manifest file
target = mkManifestTarget name;
inherit (value) enable package;
# source is a store path containing the complete manifest file
source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
[
(yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
]
# alternate the YAML doc separator (---) and extraDeploy manifests to create
# multi document YAMLs
++ (lib.concatMap (x: [
yamlDocSeparator
(mkExtraDeployManifest x)
]) value.extraDeploy)
);
};
autoDeployChartsModule = lib.types.submodule (
{ config, ... }:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
example = false;
description = ''
Whether to enable the installation of this Helm chart. Note that setting
this option to `false` will not uninstall the chart from the cluster, if
it was previously installed. Please use the the `--disable` flag or `.skip`
files to delete/disable Helm charts, as mentioned in the
[docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
'';
};
repo = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "https://kubernetes.github.io/ingress-nginx";
description = ''
The repo of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
name = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "ingress-nginx";
description = ''
The name of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
version = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "4.7.0";
description = ''
The version of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
hash = lib.mkOption {
type = lib.types.str;
example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
default = "";
description = ''
The hash of the packaged Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
package = lib.mkOption {
type = with lib.types; either path package;
example = lib.literalExpression "../my-helm-chart.tgz";
description = ''
The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
and `hash` in case of conflicts.
'';
};
targetNamespace = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "default";
example = "kube-system";
description = "The namespace in which the Helm chart gets installed.";
};
createNamespace = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether to create the target namespace if not present.";
};
values = lib.mkOption {
type = with lib.types; either path attrs;
default = { };
example = {
replicaCount = 3;
hostName = "my-host";
server = {
name = "nginx";
port = 80;
};
};
description = ''
Override default chart values via Nix expressions. This is equivalent to setting
values in a `values.yaml` file.
WARNING: The values (including secrets!) specified here are exposed unencrypted
in the world-readable nix store.
'';
};
extraDeploy = lib.mkOption {
type = with lib.types; listOf (either path attrs);
default = [ ];
example = lib.literalExpression ''
[
../manifests/my-extra-deployment.yaml
{
apiVersion = "v1";
kind = "Service";
metadata = {
name = "app-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
}
];
'';
description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
};
extraFieldDefinitions = lib.mkOption {
inherit (yamlFormat) type;
default = { };
example = {
spec = {
bootstrap = true;
helmVersion = "v2";
backOffLimit = 3;
jobImage = "custom-helm-controller:v0.0.1";
};
};
description = ''
Extra HelmChart field definitions that are merged with the rest of the HelmChart
custom resource. This can be used to set advanced fields or to overwrite
generated fields. See <https://docs.k3s.io/helm#helmchart-field-definitions>
for possible fields.
'';
};
};
config.package = lib.mkDefault (fetchHelm {
inherit (config)
repo
name
version
hash
;
});
}
);
manifestModule = lib.types.submodule (
{
name,
config,
options,
...
}:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether this manifest file should be generated.";
};
target = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "manifest.yaml";
description = ''
Name of the symlink (relative to {file}`${manifestDir}`).
Defaults to the attribute name.
'';
};
content = lib.mkOption {
type = with lib.types; nullOr (either attrs (listOf attrs));
default = null;
description = ''
Content of the manifest file. A single attribute set will
generate a single document YAML file. A list of attribute sets
will generate multiple documents separated by `---` in a single
YAML file.
'';
};
source = lib.mkOption {
type = lib.types.path;
example = lib.literalExpression "./manifests/app.yaml";
description = ''
Path of the source `.yaml` file.
'';
};
};
config = {
target = lib.mkDefault (mkManifestTarget name);
source = lib.mkIf (config.content != null) (
let
name' = "k3s-manifest-" + builtins.baseNameOf name;
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
mkSource =
value:
if builtins.isList value then
pkgs.concatText name' (
lib.concatMap (x: [
yamlDocSeparator
(yamlFormat.generate docName x)
]) value
)
else
yamlFormat.generate name' value;
in
lib.mkDerivedConfig options.content mkSource
);
};
}
);
in
{
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
# interface
options.services.k3s = {
enable = lib.mkEnableOption "k3s";
package = lib.mkPackageOption pkgs "k3s" { };
role = lib.mkOption {
description = ''
Whether k3s should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- Starts by default as a standalone server using an embedded sqlite datastore.
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
- Configure `serverAddr` to join an already-initialized HA cluster.
If it's an agent:
- `serverAddr` is required.
'';
default = "server";
type = lib.types.enum [
"server"
"agent"
];
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = ''
The k3s server to connect to.
Servers and agents need to communicate each other. Read
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
to know how to configure the firewall.
'';
example = "https://10.0.0.10:6443";
default = "";
};
clusterInit = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Initialize HA cluster using an embedded etcd datastore.
If this option is `false` and `role` is `server`
On a server that was using the default embedded sqlite backend,
enabling this option will migrate to an embedded etcd DB.
If an HA cluster using the embedded etcd datastore was already initialized,
this option has no effect.
This option only makes sense in a server that is not connecting to another server.
If you are configuring an HA cluster with an embedded etcd,
the 1st server must have `clusterInit = true`
and other servers must connect to it using `serverAddr`.
'';
};
token = lib.mkOption {
type = lib.types.str;
description = ''
The k3s token to use when connecting to a server.
WARNING: This option will expose store your token unencrypted world-readable in the nix store.
If this is undesired use the tokenFile option instead.
'';
default = "";
};
tokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "File path containing k3s token to use when connecting to the server.";
default = null;
};
extraFlags = lib.mkOption {
description = "Extra flags to pass to the k3s command.";
type = with lib.types; either str (listOf str);
default = [ ];
example = [
"--disable traefik"
"--cluster-cidr 10.24.0.0/16"
];
};
disableAgent = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Only run the server. This option only makes sense for a server.";
};
environmentFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = ''
File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
'';
default = null;
};
configPath = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
};
manifests = lib.mkOption {
type = lib.types.attrsOf manifestModule;
default = { };
example = lib.literalExpression ''
{
deployment.source = ../manifests/deployment.yaml;
my-service = {
enable = false;
target = "app-service.yaml";
content = {
apiVersion = "v1";
kind = "Service";
metadata = {
name = "app-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
};
};
nginx.content = [
{
apiVersion = "v1";
kind = "Pod";
metadata = {
name = "nginx";
labels = {
"app.kubernetes.io/name" = "MyApp";
};
};
spec = {
containers = [
{
name = "nginx";
image = "nginx:1.14.2";
ports = [
{
containerPort = 80;
name = "http-web-svc";
}
];
}
];
};
}
{
apiVersion = "v1";
kind = "Service";
metadata = {
name = "nginx-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
}
];
};
'';
description = ''
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
Note that deleting manifest files will not remove or otherwise modify the resources
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
This option only makes sense on server nodes (`role = server`).
Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
for further information.
'';
};
charts = lib.mkOption {
type = with lib.types; attrsOf (either path package);
default = { };
example = lib.literalExpression ''
nginx = ../charts/my-nginx-chart.tgz;
redis = ../charts/my-redis-chart.tgz;
'';
description = ''
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
The specified charts will only be placed on the file system and made available to the
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
'';
};
containerdConfigTemplate = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = lib.literalExpression ''
# Base K3s config
{{ template "base" . }}
# Add a custom runtime
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
BinaryName = "/path/to/custom-container-runtime"
'';
description = ''
Config template for containerd, to be placed at
`/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
'';
};
images = lib.mkOption {
type = with lib.types; listOf package;
default = [ ];
example = lib.literalExpression ''
[
(pkgs.dockerTools.pullImage {
imageName = "docker.io/bitnami/keycloak";
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
config.services.k3s.package.airgap-images
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
use, if you want to pre-provision this node with all k3s container images. This option
only makes sense on nodes with an enabled agent.
'';
};
gracefulNodeShutdown = {
enable = lib.mkEnableOption ''
graceful node shutdowns where the kubelet attempts to detect
node system shutdown and terminates pods running on the node. See the
[documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
for further information.
'';
shutdownGracePeriod = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "30s";
example = "1m30s";
description = ''
Specifies the total duration that the node should delay the shutdown by. This is the total
grace period for pod termination for both regular and critical pods.
'';
};
shutdownGracePeriodCriticalPods = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "10s";
example = "15s";
description = ''
Specifies the duration used to terminate critical pods during a node shutdown. This should be
less than `shutdownGracePeriod`.
'';
};
};
extraKubeletConfig = lib.mkOption {
type = with lib.types; attrsOf anything;
default = { };
example = {
podsPerCore = 3;
memoryThrottlingFactor = 0.69;
containerLogMaxSize = "5Mi";
};
description = ''
Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
configuration that can be configured via a file is defined by the
[KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
struct. See the
[documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
for further information.
'';
};
extraKubeProxyConfig = lib.mkOption {
type = with lib.types; attrsOf anything;
default = { };
example = {
mode = "nftables";
clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
};
description = ''
Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
configuration that can be configured via a file is defined by the
[KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
'';
};
autoDeployCharts = lib.mkOption {
type = lib.types.attrsOf autoDeployChartsModule;
apply = lib.mapAttrs mkAutoDeployChartManifest;
default = { };
example = lib.literalExpression ''
{
harbor = {
name = "harbor";
repo = "https://helm.goharbor.io";
version = "1.14.0";
hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
values = {
existingSecretAdminPassword = "harbor-admin";
expose = {
tls = {
enabled = true;
certSource = "secret";
secret.secretName = "my-tls-secret";
};
ingress = {
hosts.core = "example.com";
className = "nginx";
};
};
};
};
nginx = {
repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
version = "20.0.0";
hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
};
custom-chart = {
package = ../charts/my-chart.tgz;
values = ../values/my-values.yaml;
extraFieldDefinitions = {
spec.timeout = "60s";
};
};
}
'';
description = ''
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
attribute names that are also used in the [](#opt-services.k3s.manifests) and
[](#opt-services.k3s.charts) options. Manifests with the same name will override
auto deploying charts with the same name. Similiarly, charts with the same name will
overwrite the Helm chart contained in auto deploying charts. This option only makes
sense on server nodes (`role = server`). See the
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
'';
};
};
# implementation
config = lib.mkIf cfg.enable {
warnings =
(lib.optional (cfg.role != "server" && cfg.manifests != { })
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (duplicateManifests != [ ])
"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
)
++ (lib.optional (duplicateCharts != [ ])
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
)
++ (lib.optional (
cfg.disableAgent && cfg.images != [ ]
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
++ (lib.optional (
cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
++ (lib.optional
(cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
"k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
);
assertions = [
{
assertion = cfg.role == "agent" -> !cfg.disableAgent;
message = "k3s: disableAgent must be false if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !cfg.clusterInit;
message = "k3s: clusterInit must be false if role is 'agent'";
}
];
environment.systemPackages = [ config.services.k3s.package ];
# Use systemd-tmpfiles to activate k3s content
systemd.tmpfiles.settings."10-k3s" =
let
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
# Merge charts with charts contained in enabled auto deploying charts
helmCharts =
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
))
// cfg.charts;
# Make a systemd-tmpfiles rule for a manifest
mkManifestRule = manifest: {
name = "${manifestDir}/${manifest.target}";
value = {
"L+".argument = "${manifest.source}";
};
};
# Ensure that all chart targets have a .tgz suffix
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
# Make a systemd-tmpfiles rule for a chart
mkChartRule = target: source: {
name = "${chartDir}/${mkChartTarget target}";
value = {
"L+".argument = "${source}";
};
};
# Make a systemd-tmpfiles rule for a container image
mkImageRule = image: {
name = "${imageDir}/${image.name}";
value = {
"L+".argument = "${image}";
};
};
in
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
// (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
// (builtins.listToAttrs (map mkImageRule cfg.images))
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
${containerdConfigTemplateFile} = {
"L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
};
});
systemd.services.k3s =
let
kubeletParams =
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
})
// cfg.extraKubeletConfig;
kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
{
apiVersion = "kubelet.config.k8s.io/v1beta1";
kind = "KubeletConfiguration";
}
// kubeletParams
);
kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
{
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
kind = "KubeProxyConfiguration";
}
// cfg.extraKubeProxyConfig
);
in
{
description = "k3s service";
after = [
"firewall.service"
"network-online.target"
];
wants = [
"firewall.service"
"network-online.target"
];
wantedBy = [ "multi-user.target" ];
path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
serviceConfig = {
# See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
Type = if cfg.role == "agent" then "exec" else "notify";
KillMode = "process";
Delegate = "yes";
Restart = "always";
RestartSec = "5s";
LimitNOFILE = 1048576;
LimitNPROC = "infinity";
LimitCORE = "infinity";
TasksMax = "infinity";
EnvironmentFile = cfg.environmentFile;
ExecStart = lib.concatStringsSep " \\\n " (
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
++ (lib.optional cfg.clusterInit "--cluster-init")
++ (lib.optional cfg.disableAgent "--disable-agent")
++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
++ (lib.flatten cfg.extraFlags)
);
};
};
};
meta.maintainers = lib.teams.k3s.members;
}

966
nixos/modules/services/cluster/rancher/default.nix Normal file
View File

@@ -0,0 +1,966 @@
{
config,
lib,
pkgs,
...
}:
let
mkRancherModule =
{
# name used in paths/bin names/etc, e.g. k3s
name,
# systemd service name
serviceName ? name,
# extra flags to pass to the binary before user-defined extraFlags
extraBinFlags ? [ ],
# generate manifests as JSON rather than YAML, see rke2.nix
jsonManifests ? false,
# which port on the local node hosts content placed in ${staticContentChartDir} on /static/
# if null, it's assumed the content can be accessed via https://%{KUBERNETES_API}%/static/
staticContentPort ? null,
}:
let
cfg = config.services.${name};
# Paths defined here are passed to the downstream modules as `paths`
manifestDir = "/var/lib/rancher/${name}/server/manifests";
imageDir = "/var/lib/rancher/${name}/agent/images";
containerdConfigTemplateFile = "/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl";
staticContentChartDir = "/var/lib/rancher/${name}/server/static/charts";
manifestFormat = if jsonManifests then pkgs.formats.json { } else pkgs.formats.yaml { };
# Manifests need a valid suffix to be respected
mkManifestTarget =
name:
if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name || lib.hasSuffix ".json" name) then
name
else if jsonManifests then
name + ".json"
else
name + ".yaml";
# Returns a path to the final manifest file
mkManifestSource =
name: manifests:
manifestFormat.generate name (
if builtins.isList manifests then
{
apiVersion = "v1";
kind = "List";
items = manifests;
}
else
manifests
);
# Produces a list containing all duplicate manifest names
duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.manifests
);
# Produces a list containing all duplicate chart names
duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
builtins.attrNames cfg.charts
);
# Converts YAML -> JSON -> Nix
fromYaml =
path:
builtins.fromJSON (
builtins.readFile (
pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ pkgs.yq-go ]; } ''
yq --no-colors --output-format json ${path} > $out
''
)
);
# Replace prefixes and characters that are problematic in file names
cleanHelmChartName =
name:
let
woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
in
lib.replaceStrings
[
"/"
":"
]
[
"-"
"-"
]
woPrefix;
# Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
fetchHelm =
{
name,
repo,
version,
hash ? lib.fakeHash,
}:
let
isOci = lib.hasPrefix "oci://" repo;
pullCmd = if isOci then repo else "--repo ${repo} ${name}";
name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
in
pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
{
inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
nativeBuildInputs = with pkgs; [
kubernetes-helm
cacert
# Helm requires HOME to refer to a writable dir
writableTmpDirAsHomeHook
];
}
''
helm pull ${pullCmd} --version ${version}
mv ./*.tgz $out
'';
# Returns the path to a YAML manifest file
mkExtraDeployManifest =
x:
# x is a derivation that provides a YAML file
if lib.isDerivation x then
x.outPath
# x is an attribute set that needs to be converted to a YAML file
else if builtins.isAttrs x then
(manifestFormat.generate "extra-deploy-chart-manifest" x)
# assume x is a path to a YAML file
else
x;
# Generate a HelmChart custom resource.
mkHelmChartCR =
name: value:
let
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
# use JSON for values as it's a subset of YAML and understood by the rancher Helm controller
valuesContent = builtins.toJSON chartValues;
in
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
# values
lib.recursiveUpdate {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChart";
metadata = {
inherit name;
namespace = "kube-system";
};
spec = {
inherit valuesContent;
inherit (value) targetNamespace createNamespace;
chart =
if staticContentPort == null then
"https://%{KUBERNETES_API}%/static/charts/${name}.tgz"
else
"https://localhost:${toString staticContentPort}/static/charts/${name}.tgz";
bootstrap = staticContentPort != null; # needed for host network access
};
} value.extraFieldDefinitions;
# Generate a HelmChart custom resource together with extraDeploy manifests.
mkAutoDeployChartManifest = name: value: {
# target is the final name of the link created for the manifest file
target = mkManifestTarget name;
inherit (value) enable package;
# source is a store path containing the complete manifest file
source = mkManifestSource "auto-deploy-chart-${name}" (
lib.singleton (mkHelmChartCR name value)
++ builtins.map (x: fromYaml (mkExtraDeployManifest x)) value.extraDeploy
);
};
autoDeployChartsModule = lib.types.submodule (
{ config, ... }:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
example = false;
description = ''
Whether to enable the installation of this Helm chart. Note that setting
this option to `false` will not uninstall the chart from the cluster, if
it was previously installed. Please use the the `--disable` flag or `.skip`
files to delete/disable Helm charts, as mentioned in the
[docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
'';
};
repo = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "https://kubernetes.github.io/ingress-nginx";
description = ''
The repo of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
name = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "ingress-nginx";
description = ''
The name of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
version = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "4.7.0";
description = ''
The version of the Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
hash = lib.mkOption {
type = lib.types.str;
example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
default = "";
description = ''
The hash of the packaged Helm chart. Only has an effect if `package` is not set.
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
filesystem.
'';
};
package = lib.mkOption {
type = with lib.types; either path package;
example = lib.literalExpression "../my-helm-chart.tgz";
description = ''
The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
and `hash` in case of conflicts.
'';
};
targetNamespace = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "default";
example = "kube-system";
description = "The namespace in which the Helm chart gets installed.";
};
createNamespace = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether to create the target namespace if not present.";
};
values = lib.mkOption {
type = with lib.types; either path attrs;
default = { };
example = {
replicaCount = 3;
hostName = "my-host";
server = {
name = "nginx";
port = 80;
};
};
description = ''
Override default chart values via Nix expressions. This is equivalent to setting
values in a `values.yaml` file.
**WARNING**: The values (including secrets!) specified here are exposed unencrypted
in the world-readable nix store.
'';
};
extraDeploy = lib.mkOption {
type = with lib.types; listOf (either path attrs);
default = [ ];
example = lib.literalExpression ''
[
../manifests/my-extra-deployment.yaml
{
apiVersion = "v1";
kind = "Service";
metadata = {
name = "app-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
}
];
'';
description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
};
extraFieldDefinitions = lib.mkOption {
inherit (manifestFormat) type;
default = { };
example = {
spec = {
bootstrap = true;
helmVersion = "v2";
backOffLimit = 3;
jobImage = "custom-helm-controller:v0.0.1";
};
};
description = ''
Extra HelmChart field definitions that are merged with the rest of the HelmChart
custom resource. This can be used to set advanced fields or to overwrite
generated fields. See <https://docs.${name}.io/helm#helmchart-field-definitions>
for possible fields.
'';
};
};
config.package = lib.mkDefault (fetchHelm {
inherit (config)
repo
name
version
hash
;
});
}
);
manifestModule = lib.types.submodule (
{
name,
config,
options,
...
}:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether this manifest file should be generated.";
};
target = lib.mkOption {
type = lib.types.nonEmptyStr;
example = "manifest.yaml";
description = ''
Name of the symlink (relative to {file}`${manifestDir}`).
Defaults to the attribute name.
'';
};
content = lib.mkOption {
type = with lib.types; nullOr (either attrs (listOf attrs));
default = null;
description = ''
Content of the manifest file. A single attribute set will
generate a single document YAML file. A list of attribute sets
will generate multiple documents separated by `---` in a single
YAML file.
'';
};
source = lib.mkOption {
type = lib.types.path;
example = lib.literalExpression "./manifests/app.yaml";
description = ''
Path of the source `.yaml` file.
'';
};
};
config = {
target = lib.mkDefault (mkManifestTarget name);
source = lib.mkIf (config.content != null) (
let
name' = "${name}-manifest-" + builtins.baseNameOf name;
mkSource = mkManifestSource name';
in
lib.mkDerivedConfig options.content mkSource
);
};
}
);
in
{
paths = {
inherit
manifestDir
imageDir
containerdConfigTemplateFile
staticContentChartDir
;
};
# interface
options = {
enable = lib.mkEnableOption name;
package = lib.mkPackageOption pkgs name { };
role = lib.mkOption {
description = "Whether ${name} should run as a server or agent.";
default = "server";
type = lib.types.enum [
"server"
"agent"
];
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = "The ${name} server to connect to, used to join a cluster.";
example = "https://10.0.0.10:6443";
default = "";
};
token = lib.mkOption {
type = lib.types.str;
description = ''
The ${name} token to use when connecting to a server.
**WARNING**: This option will expose your token unencrypted in the world-readable nix store.
If this is undesired use the tokenFile option instead.
'';
default = "";
};
tokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "File path containing the ${name} token to use when connecting to a server.";
default = null;
};
agentToken = lib.mkOption {
type = lib.types.str;
description = ''
The ${name} token agents can use to connect to the server.
This option only makes sense on server nodes (`role = server`).
**WARNING**: This option will expose your token unencrypted in the world-readable nix store.
If this is undesired use the tokenFile option instead.
'';
default = "";
};
agentTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = ''
File path containing the ${name} token agents can use to connect to the server.
This option only makes sense on server nodes (`role = server`).
'';
default = null;
};
extraFlags = lib.mkOption {
description = "Extra flags to pass to the ${name} command.";
type = with lib.types; either str (listOf str);
default = [ ];
example = [
"--etcd-expose-metrics"
"--cluster-cidr 10.24.0.0/16"
];
};
environmentFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = ''
File path containing environment variables for configuring the ${name} service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
'';
default = null;
};
configPath = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "File path containing the ${name} YAML config. This is useful when the config is generated (for example on boot).";
};
disable = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Disable default components via the `--disable` flag.";
default = [ ];
};
nodeName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Node name.";
default = null;
};
nodeLabel = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Registering and starting kubelet with set of labels.";
default = [ ];
};
nodeTaint = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Registering kubelet with set of taints.";
default = [ ];
};
nodeIP = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "IPv4/IPv6 addresses to advertise for node.";
default = null;
};
selinux = lib.mkOption {
type = lib.types.bool;
description = "Enable SELinux in containerd.";
default = false;
};
manifests = lib.mkOption {
type = lib.types.attrsOf manifestModule;
default = { };
example = lib.literalExpression ''
{
deployment.source = ../manifests/deployment.yaml;
my-service = {
enable = false;
target = "app-service.yaml";
content = {
apiVersion = "v1";
kind = "Service";
metadata = {
name = "app-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
};
};
nginx.content = [
{
apiVersion = "v1";
kind = "Pod";
metadata = {
name = "nginx";
labels = {
"app.kubernetes.io/name" = "MyApp";
};
};
spec = {
containers = [
{
name = "nginx";
image = "nginx:1.14.2";
ports = [
{
containerPort = 80;
name = "http-web-svc";
}
];
}
];
};
}
{
apiVersion = "v1";
kind = "Service";
metadata = {
name = "nginx-service";
};
spec = {
selector = {
"app.kubernetes.io/name" = "MyApp";
};
ports = [
{
name = "name-of-service-port";
protocol = "TCP";
port = 80;
targetPort = "http-web-svc";
}
];
};
}
];
};
'';
description = ''
Auto-deploying manifests that are linked to {file}`${manifestDir}` before ${name} starts.
Note that deleting manifest files will not remove or otherwise modify the resources
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
This option only makes sense on server nodes (`role = server`).
Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
for further information.
**WARNING**: If you have multiple server nodes, and set this option on more than one server,
it is your responsibility to ensure that files stay in sync across those nodes. AddOn content is
not synced between nodes, and ${name} cannot guarantee correct behavior if different servers attempt
to deploy conflicting manifests.
'';
};
containerdConfigTemplate = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = lib.literalExpression ''
# Base config
{{ template "base" . }}
# Add a custom runtime
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
BinaryName = "/path/to/custom-container-runtime"
'';
description = ''
Config template for containerd, to be placed at
`/var/lib/rancher/${name}/agent/etc/containerd/config.toml.tmpl`.
See the docs on [configuring containerd](https://docs.${name}.io/advanced#configuring-containerd).
'';
};
images = lib.mkOption {
type = with lib.types; listOf package;
default = [ ];
example = lib.literalExpression ''
[
(pkgs.dockerTools.pullImage {
imageName = "docker.io/bitnami/keycloak";
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${imageDir}` before ${name} starts and are consequently imported
by the ${name} agent. This option only makes sense on nodes with an enabled agent.
'';
};
gracefulNodeShutdown = {
enable = lib.mkEnableOption ''
graceful node shutdowns where the kubelet attempts to detect
node system shutdown and terminates pods running on the node. See the
[documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
for further information.
'';
shutdownGracePeriod = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "30s";
example = "1m30s";
description = ''
Specifies the total duration that the node should delay the shutdown by. This is the total
grace period for pod termination for both regular and critical pods.
'';
};
shutdownGracePeriodCriticalPods = lib.mkOption {
type = lib.types.nonEmptyStr;
default = "10s";
example = "15s";
description = ''
Specifies the duration used to terminate critical pods during a node shutdown. This should be
less than `shutdownGracePeriod`.
'';
};
};
extraKubeletConfig = lib.mkOption {
type = with lib.types; attrsOf anything;
default = { };
example = {
podsPerCore = 3;
memoryThrottlingFactor = 0.69;
containerLogMaxSize = "5Mi";
};
description = ''
Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
configuration that can be configured via a file is defined by the
[KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
struct. See the
[documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
for further information.
'';
};
extraKubeProxyConfig = lib.mkOption {
type = with lib.types; attrsOf anything;
default = { };
example = {
mode = "nftables";
clientConnection.kubeconfig = "/var/lib/rancher/${name}/agent/kubeproxy.kubeconfig";
};
description = ''
Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
configuration that can be configured via a file is defined by the
[KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
struct. Note that the kubeconfig param will be overriden by `clientConnection.kubeconfig`, so you must
set the `clientConnection.kubeconfig` option if you want to use `extraKubeProxyConfig`.
'';
};
autoDeployCharts = lib.mkOption {
type = lib.types.attrsOf autoDeployChartsModule;
apply = lib.mapAttrs mkAutoDeployChartManifest;
default = { };
example = lib.literalExpression ''
{
harbor = {
name = "harbor";
repo = "https://helm.goharbor.io";
version = "1.14.0";
hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
values = {
existingSecretAdminPassword = "harbor-admin";
expose = {
tls = {
enabled = true;
certSource = "secret";
secret.secretName = "my-tls-secret";
};
ingress = {
hosts.core = "example.com";
className = "nginx";
};
};
};
};
nginx = {
repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
version = "20.0.0";
hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
};
custom-chart = {
package = ../charts/my-chart.tgz;
values = ../values/my-values.yaml;
extraFieldDefinitions = {
spec.timeout = "60s";
};
};
}
'';
description = ''
Auto deploying Helm charts that are installed by the ${name} Helm controller. Avoid using
attribute names that are also used in the [](#opt-services.${name}.manifests) and
[](#opt-services.${name}.charts) options. Manifests with the same name will override
auto deploying charts with the same name.
This option only makes sense on server nodes (`role = server`). See the
[${name} Helm documentation](https://docs.${name}.io/helm) for further information.
**WARNING**: If you have multiple server nodes, and set this option on more than one server,
it is your responsibility to ensure that files stay in sync across those nodes. AddOn content is
not synced between nodes, and ${name} cannot guarantee correct behavior if different servers attempt
to deploy conflicting manifests.
'';
};
charts = lib.mkOption {
type = with lib.types; attrsOf (either path package);
default = { };
example = lib.literalExpression ''
nginx = ../charts/my-nginx-chart.tgz;
redis = ../charts/my-redis-chart.tgz;
'';
description = ''
Packaged Helm charts that are linked to {file}`${staticContentChartDir}` before ${name} starts.
The attribute name will be used as the link target (relative to {file}`${staticContentChartDir}`).
The specified charts will only be placed on the file system and made available via ${
if staticContentPort == null then
"the Kubernetes APIServer from within the cluster"
else
"port ${toString staticContentPort} on server nodes"
}. See the [](#opt-services.${name}.autoDeployCharts) option and the
[${name} Helm controller docs](https://docs.${name}.io/helm#using-the-helm-controller)
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
'';
};
};
# implementation
config = {
warnings =
(lib.optional (cfg.role != "server" && cfg.manifests != { })
"${name}: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
"${name}: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (duplicateManifests != [ ])
"${name}: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
)
++ (lib.optional (duplicateCharts != [ ])
"${name}: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
)
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
"${name}: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
)
++ (lib.optional (
cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
) "${name}: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
++ (lib.optional
(cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
"${name}: token, tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
)
++ (lib.optional (
cfg.role == "agent" && !(cfg.agentTokenFile != null || cfg.agentToken != "")
) "${name}: agentToken and agentToken should not be set if role is 'agent'");
environment.systemPackages = [ config.services.${name}.package ];
# Use systemd-tmpfiles to activate content
systemd.tmpfiles.settings."10-${name}" =
let
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
# Make a systemd-tmpfiles rule for a manifest
mkManifestRule = manifest: {
name = "${manifestDir}/${manifest.target}";
value = {
"L+".argument = "${manifest.source}";
};
};
# Make a systemd-tmpfiles rule for a container image
mkImageRule = image: {
name = "${imageDir}/${image.name}";
value = {
"L+".argument = "${image}";
};
};
# Merge charts with charts contained in enabled auto deploying charts
helmCharts =
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
))
// cfg.charts;
# Ensure that all chart targets have a .tgz suffix
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
# Make a systemd-tmpfiles rule for a chart
mkChartRule = target: source: {
name = "${staticContentChartDir}/${mkChartTarget target}";
value = {
"L+".argument = "${source}";
};
};
in
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
// (builtins.listToAttrs (map mkImageRule cfg.images))
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
${containerdConfigTemplateFile} = {
"L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
};
})
// (lib.mapAttrs' mkChartRule helmCharts);
systemd.services.${serviceName} =
let
kubeletParams =
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
})
// cfg.extraKubeletConfig;
kubeletConfig = manifestFormat.generate "${name}-kubelet-config" (
{
apiVersion = "kubelet.config.k8s.io/v1beta1";
kind = "KubeletConfiguration";
}
// kubeletParams
);
kubeProxyConfig = manifestFormat.generate "${name}-kubeProxy-config" (
{
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
kind = "KubeProxyConfiguration";
}
// cfg.extraKubeProxyConfig
);
in
{
description = "${name} service";
after = [
"firewall.service"
"network-online.target"
];
wants = [
"firewall.service"
"network-online.target"
];
wantedBy = [ "multi-user.target" ];
path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
serviceConfig = {
# See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
Type = if cfg.role == "agent" then "exec" else "notify";
KillMode = "process";
Delegate = "yes";
Restart = "always";
RestartSec = "5s";
LimitNOFILE = 1048576;
LimitNPROC = "infinity";
LimitCORE = "infinity";
TasksMax = "infinity";
TimeoutStartSec = 0;
EnvironmentFile = cfg.environmentFile;
ExecStart = lib.concatStringsSep " \\\n " (
[ "${cfg.package}/bin/${name} ${cfg.role}" ]
++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
++ (lib.optional (cfg.agentToken != "") "--agent-token ${cfg.agentToken}")
++ (lib.optional (cfg.agentTokenFile != null) "--agent-token-file ${cfg.agentTokenFile}")
++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
++ (map (d: "--disable=${d}") cfg.disable)
++ (lib.optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}")
++ (lib.optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel))
++ (lib.optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint))
++ (lib.optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}")
++ (lib.optional cfg.selinux "--selinux")
++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
++ extraBinFlags
++ (lib.flatten cfg.extraFlags)
);
};
};
};
};
in
{
imports =
# pass mkRancherModule explicitly instead of via
# _modules.args to prevent infinite recursion
let
args = {
inherit config lib;
inherit mkRancherModule;
};
in
[
(import ./k3s.nix args)
(import ./rke2.nix args)
];
meta.maintainers = pkgs.rke2.meta.maintainers ++ lib.teams.k3s.members;
}

137
nixos/modules/services/cluster/rancher/k3s.nix Normal file
View File

@@ -0,0 +1,137 @@
{
config,
lib,
mkRancherModule,
...
}:
let
cfg = config.services.k3s;
baseModule = mkRancherModule {
name = "k3s";
extraBinFlags =
(lib.optional cfg.clusterInit "--cluster-init")
++ (lib.optional cfg.disableAgent "--disable-agent");
};
removeOption =
config: instruction:
lib.mkRemovedOptionModule (
[
"services"
"k3s"
]
++ config
) instruction;
in
{
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
# interface
options.services.k3s = lib.recursiveUpdate baseModule.options {
# option overrides
role.description = ''
Whether k3s should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- Starts by default as a standalone server using an embedded sqlite datastore.
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
- Configure `serverAddr` to join an already-initialized HA cluster.
If it's an agent:
- `serverAddr` is required.
'';
serverAddr.description = ''
The k3s server to connect to.
Servers and agents need to communicate each other. Read
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
to know how to configure the firewall.
'';
disable.description = ''
Disable default components, see the [K3s documentation](https://docs.k3s.io/installation/packaged-components#using-the---disable-flag).
'';
images = {
example = lib.literalExpression ''
[
(pkgs.dockerTools.pullImage {
imageName = "docker.io/bitnami/keycloak";
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
config.services.k3s.package.airgap-images
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${baseModule.paths.imageDir}` before k3s starts and are consequently imported
by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
use, if you want to pre-provision this node with all k3s container images. This option
only makes sense on nodes with an enabled agent.
'';
};
# k3s-specific options
clusterInit = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Initialize HA cluster using an embedded etcd datastore.
If this option is `false` and `role` is `server`
On a server that was using the default embedded sqlite backend,
enabling this option will migrate to an embedded etcd DB.
If an HA cluster using the embedded etcd datastore was already initialized,
this option has no effect.
This option only makes sense in a server that is not connecting to another server.
If you are configuring an HA cluster with an embedded etcd,
the 1st server must have `clusterInit = true`
and other servers must connect to it using `serverAddr`.
'';
};
disableAgent = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Only run the server. This option only makes sense for a server.";
};
};
# implementation
config = lib.mkIf cfg.enable (
lib.recursiveUpdate baseModule.config {
warnings = (
lib.optional (
cfg.disableAgent && cfg.images != [ ]
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node."
);
assertions = [
{
assertion = cfg.role == "agent" -> !cfg.disableAgent;
message = "k3s: disableAgent must be false if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !cfg.clusterInit;
message = "k3s: clusterInit must be false if role is 'agent'";
}
];
}
);
}

160
nixos/modules/services/cluster/rancher/rke2.nix Normal file
View File

@@ -0,0 +1,160 @@
{
config,
lib,
mkRancherModule,
...
}:
let
cfg = config.services.rke2;
baseModule = mkRancherModule {
name = "rke2";
serviceName = "rke2-${cfg.role}"; # upstream default, used by rke2-killall.sh
extraBinFlags =
(lib.optional (cfg.cni != null) "--cni=${cfg.cni}")
++ (lib.optional cfg.cisHardening "--profile=${
if lib.versionAtLeast cfg.package.version "1.25" then
"cis"
else if lib.versionAtLeast cfg.package.version "1.23" then
"cis-1.23"
else
"cis-1.6"
}");
# RKE2 sometimes tries opening YAML manifests on start with O_RDWR, which we can't support
# without ugly workarounds since they're linked from the read-only /nix/store.
# https://github.com/rancher/rke2/blob/fa7ed3a87055830924d05009a1071acfbbfbcc2c/pkg/bootstrap/bootstrap.go#L355
jsonManifests = true;
# see https://github.com/rancher/rke2/issues/224
# not all charts can be base64-encoded into chartContent due to
# https://github.com/k3s-io/helm-controller/issues/267
staticContentPort = 9345;
};
in
{
# interface
options.services.rke2 = lib.recursiveUpdate baseModule.options {
# option overrides
role.description = ''
Whether rke2 should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- All options can be set.
If it's an agent:
- `serverAddr` is required.
- `token` or `tokenFile` is required.
- `agentToken`, `agentTokenFile`, `disable` and `cni` should not be set.
'';
disable.description = ''
Disable default components, see the [RKE2 documentation](https://docs.rke2.io/install/packaged_components#using-the---disable-flag).
'';
images = {
example = lib.literalExpression ''
[
(pkgs.dockerTools.pullImage {
imageName = "docker.io/bitnami/keycloak";
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
finalImageTag = "21.1.2-debian-11-r0";
})
config.services.rke2.package.images-core-linux-amd64-tar-zst
config.services.rke2.package.images-canal-linux-amd64-tar-zst
]
'';
description = ''
List of derivations that provide container images.
All images are linked to {file}`${baseModule.paths.imageDir}` before rke2 starts and are consequently imported
by the rke2 agent. Consider importing the rke2 core and CNI image archives of the rke2 package in
use, if you want to pre-provision this node with all rke2 container images. For a full list of available airgap images, check the
[source](https://github.com/NixOS/nixpkgs/blob/c8a1939887ee6e5f5aae29ce97321c0d83165f7d/pkgs/applications/networking/cluster/rke2/1_32/images-versions.json).
of the rke2 package in use.
'';
};
# rke2-specific options
cni = lib.mkOption {
type =
with lib.types;
nullOr (enum [
"none"
"canal"
"cilium"
"calico"
"flannel"
]);
description = ''
CNI plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`.
All CNI plugins get installed via a helm chart after the main components are up and running
and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm).
[Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options)
> **WARNING**: Flannel support in RKE2 is currently experimental.
'';
default = null;
};
cisHardening = lib.mkOption {
type = lib.types.bool;
description = ''
Enable CIS Hardening for RKE2.
The OS-level configuration options required to pass the CIS benchmark are enabled by default.
This option only creates the `etcd` user and group, and passes the `--profile=cis` flag to RKE2.
Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide).
'';
default = false;
};
};
# implementation
config = lib.mkIf cfg.enable (
lib.recursiveUpdate baseModule.config {
warnings = (
lib.optional (
cfg.role == "agent" && cfg.cni != null
) "rke2: cni should not be set if role is 'agent'"
);
# Configure NetworkManager to ignore CNI network interfaces.
# See: https://docs.rke2.io/known_issues#networkmanager
environment.etc."NetworkManager/conf.d/rke2-canal.conf" = {
enable = config.networking.networkmanager.enable;
text = ''
[keyfile]
unmanaged-devices=interface-name:flannel*;interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico;interface-name:wireguard.cali;interface-name:wg-v6.cali
'';
};
# CIS hardening
# https://docs.rke2.io/security/hardening_guide#kernel-parameters
# https://github.com/rancher/rke2/blob/ef0fc7aa9d3bbaa95ce9b1895972488cbd92e302/bundle/share/rke2/rke2-cis-sysctl.conf
boot.kernel.sysctl = {
"vm.panic_on_oom" = 0;
"vm.overcommit_memory" = 1;
"kernel.panic" = 10;
"kernel.panic_on_oops" = 1;
};
# https://docs.rke2.io/security/hardening_guide#etcd-is-configured-properly
users = lib.mkIf cfg.cisHardening {
users.etcd = {
isSystemUser = true;
group = "etcd";
};
groups.etcd = { };
};
}
);
}

338
nixos/modules/services/cluster/rke2/default.nix
View File

@@ -1,338 +0,0 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.rke2;
in
{
imports = [ ];
options.services.rke2 = {
enable = lib.mkEnableOption "rke2";
package = lib.mkPackageOption pkgs "rke2" { };
role = lib.mkOption {
type = lib.types.enum [
"server"
"agent"
];
description = ''
Whether rke2 should run as a server or agent.
If it's a server:
- By default it also runs workloads as an agent.
- any optionals is allowed.
If it's an agent:
- `serverAddr` is required.
- `token` or `tokenFile` is required.
- `agentToken` or `agentTokenFile` or `disable` or `cni` are not allowed.
'';
default = "server";
};
configPath = lib.mkOption {
type = lib.types.path;
description = "Load configuration from FILE.";
default = "/etc/rancher/rke2/config.yaml";
};
debug = lib.mkOption {
type = lib.types.bool;
description = "Turn on debug logs.";
default = false;
};
dataDir = lib.mkOption {
type = lib.types.path;
description = "The folder to hold state in.";
default = "/var/lib/rancher/rke2";
};
token = lib.mkOption {
type = lib.types.str;
description = ''
Shared secret used to join a server or agent to a cluster.
> WARNING: This option will expose store your token unencrypted world-readable in the nix store.
If this is undesired use the `tokenFile` option instead.
'';
default = "";
};
tokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "File path containing rke2 token to use when connecting to the server.";
default = null;
};
disable = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Do not deploy packaged components and delete any deployed components.";
default = [ ];
};
nodeName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Node name.";
default = null;
};
nodeLabel = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Registering and starting kubelet with set of labels.";
default = [ ];
};
nodeTaint = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Registering kubelet with set of taints.";
default = [ ];
};
nodeIP = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "IPv4/IPv6 addresses to advertise for node.";
default = null;
};
agentToken = lib.mkOption {
type = lib.types.str;
description = ''
Shared secret used to join agents to the cluster, but not servers.
> **WARNING**: This option will expose store your token unencrypted world-readable in the nix store.
If this is undesired use the `agentTokenFile` option instead.
'';
default = "";
};
agentTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "File path containing rke2 agent token to use when connecting to the server.";
default = null;
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = "The rke2 server to connect to, used to join a cluster.";
example = "https://10.0.0.10:6443";
default = "";
};
selinux = lib.mkOption {
type = lib.types.bool;
description = "Enable SELinux in containerd.";
default = false;
};
cni = lib.mkOption {
type = lib.types.enum [
"none"
"canal"
"cilium"
"calico"
"flannel"
];
description = ''
CNI Plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`.
All CNI plugins get installed via a helm chart after the main components are up and running
and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm).
[Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options)
> **WARNING**: Flannel support in RKE2 is currently experimental.
'';
default = "canal";
};
cisHardening = lib.mkOption {
type = lib.types.bool;
description = ''
Enable CIS Hardening for RKE2.
It will set the configurations and controls required to address Kubernetes benchmark controls
from the Center for Internet Security (CIS).
Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide).
> **NOTICE**:
>
> You may need restart the `systemd-sysctl` muaually by:
>
> ```shell
> sudo systemctl restart systemd-sysctl
> ```
'';
default = false;
};
extraFlags = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
Extra flags to pass to the rke2 service/agent.
Here you can find all the available flags:
- [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
- [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
'';
example = [
"--disable-kube-proxy"
"--cluster-cidr=10.24.0.0/16"
];
default = [ ];
};
environmentVars = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
description = ''
Environment variables for configuring the rke2 service/agent.
Here you can find all the available environment variables:
- [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
- [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
Besides the options above, you can also active environment variables by edit/create those files:
- `/etc/default/rke2`
- `/etc/sysconfig/rke2`
- `/usr/local/lib/systemd/system/rke2.env`
'';
# See: https://github.com/rancher/rke2/blob/master/bundle/lib/systemd/system/rke2-server.env#L1
default = {
HOME = "/root";
};
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.serverAddr != "");
message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
}
{
assertion =
cfg.role == "agent"
-> (builtins.pathExists cfg.configPath || cfg.tokenFile != null || cfg.token != "");
message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !(cfg.agentTokenFile != null || cfg.agentToken != "");
message = "agentToken or agentTokenFile should NOT be set if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !(cfg.disable != [ ]);
message = "disable should not be set if role is 'agent'";
}
{
assertion = cfg.role == "agent" -> !(cfg.cni != "canal");
message = "cni should not be set if role is 'agent'";
}
];
environment.systemPackages = [ config.services.rke2.package ];
# To configure NetworkManager to ignore calico/flannel related network interfaces.
# See: https://docs.rke2.io/known_issues#networkmanager
environment.etc."NetworkManager/conf.d/rke2-canal.conf" = {
enable = config.networking.networkmanager.enable;
text = ''
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:flannel*
'';
};
# See: https://docs.rke2.io/security/hardening_guide#set-kernel-parameters
boot.kernel.sysctl = lib.mkIf cfg.cisHardening {
"vm.panic_on_oom" = 0;
"vm.overcommit_memory" = 1;
"kernel.panic" = 10;
"kernel.panic_on_oops" = 1;
};
systemd.services."rke2-${cfg.role}" = {
description = "Rancher Kubernetes Engine v2";
documentation = [ "https://github.com/rancher/rke2#readme" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = if cfg.role == "agent" then "exec" else "notify";
EnvironmentFile = [
"-/etc/default/%N"
"-/etc/sysconfig/%N"
"-/usr/local/lib/systemd/system/%N.env"
];
Environment = lib.mapAttrsToList (k: v: "${k}=${v}") cfg.environmentVars;
KillMode = "process";
Delegate = "yes";
LimitNOFILE = 1048576;
LimitNPROC = "infinity";
LimitCORE = "infinity";
TasksMax = "infinity";
TimeoutStartSec = 0;
Restart = "always";
RestartSec = "5s";
ExecStartPre = [
# There is a conflict between RKE2 and `nm-cloud-setup.service`. This service add a routing table that
# interfere with the CNI plugin's configuration. This script checks if the service is enabled and if so,
# failed the RKE2 start.
# See: https://github.com/rancher/rke2/issues/1053
(pkgs.writeScript "check-nm-cloud-setup.sh" ''
#! ${pkgs.runtimeShell}
set -x
! /run/current-system/systemd/bin/systemctl is-enabled --quiet nm-cloud-setup.service
'')
"-${pkgs.kmod}/bin/modprobe br_netfilter"
"-${pkgs.kmod}/bin/modprobe overlay"
];
ExecStart = "${cfg.package}/bin/rke2 '${cfg.role}' ${
lib.escapeShellArgs (
(lib.optional (cfg.configPath != "/etc/rancher/rke2/config.yaml") "--config=${cfg.configPath}")
++ (lib.optional cfg.debug "--debug")
++ (lib.optional (cfg.dataDir != "/var/lib/rancher/rke2") "--data-dir=${cfg.dataDir}")
++ (lib.optional (cfg.token != "") "--token=${cfg.token}")
++ (lib.optional (cfg.tokenFile != null) "--token-file=${cfg.tokenFile}")
++ (lib.optionals (cfg.role == "server" && cfg.disable != [ ]) (
map (d: "--disable=${d}") cfg.disable
))
++ (lib.optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}")
++ (lib.optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel))
++ (lib.optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint))
++ (lib.optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}")
++ (lib.optional (cfg.role == "server" && cfg.agentToken != "") "--agent-token=${cfg.agentToken}")
++ (lib.optional (
cfg.role == "server" && cfg.agentTokenFile != null
) "--agent-token-file=${cfg.agentTokenFile}")
++ (lib.optional (cfg.serverAddr != "") "--server=${cfg.serverAddr}")
++ (lib.optional cfg.selinux "--selinux")
++ (lib.optional (cfg.role == "server" && cfg.cni != "canal") "--cni=${cfg.cni}")
++ (lib.optional cfg.cisHardening "--profile=${
if cfg.package.version >= "1.25" then "cis-1.23" else "cis-1.6"
}")
++ cfg.extraFlags
)
}";
ExecStopPost =
let
killProcess = pkgs.writeScript "kill-process.sh" ''
#! ${pkgs.runtimeShell}
/run/current-system/systemd/bin/systemd-cgls /system.slice/$1 | \
${pkgs.gnugrep}/bin/grep -Eo '[0-9]+ (containerd|kubelet)' | \
${pkgs.gawk}/bin/awk '{print $1}' | \
${pkgs.findutils}/bin/xargs -r ${pkgs.util-linux}/bin/kill
'';
in
"-${killProcess} %n";
};
};
};
}

2
nixos/tests/k3s/auto-deploy-charts.nix
View File

@@ -135,7 +135,7 @@ import ../make-test-python.nix (
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/values-file.yaml")
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
# check that the timeout is set correctly, select only the first doc in advanced.yaml
advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
advancedManifest = json.loads(machine.succeed("yq -o json '.items[0]' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
t.assertEqual(advancedManifest["spec"]["timeout"], "69s", "unexpected value for spec.timeout")
# wait for test jobs to complete
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)

132
nixos/tests/rke2/multi-node.nix
View File

@@ -31,46 +31,21 @@ import ../make-test-python.nix (
];
};
};
# A daemonset that responds 'hello' on port 8000
networkTestDaemonset = pkgs.writeText "test.yml" ''
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: test
labels:
name: test
spec:
selector:
matchLabels:
name: test
template:
metadata:
labels:
name: test
spec:
containers:
- name: test
image: test.local/hello:local
imagePullPolicy: Never
resources:
limits:
memory: 20Mi
command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo hello"]
'';
tokenFile = pkgs.writeText "token" "p@s$w0rd";
agentTokenFile = pkgs.writeText "agent-token" "agentP@s$w0rd";
# Let flannel use eth1 to enable inter-node communication in tests
canalConfig = pkgs.writeText "rke2-canal-config.yaml" ''
apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: rke2-canal
namespace: kube-system
spec:
valuesContent: |-
flannel:
iface: "eth1"
'';
canalConfig = {
apiVersion = "helm.cattle.io/v1";
kind = "HelmChartConfig";
metadata = {
name = "rke2-canal";
namespace = "kube-system";
};
# spec.valuesContent needs to a string, either json or yaml
spec.valuesContent = builtins.toJSON {
flannel.iface = "eth1";
};
};
in
{
name = "${rke2.name}-multi-node";
@@ -85,23 +60,6 @@ import ../make-test-python.nix (
...
}:
{
# Setup image archives to be imported by rke2
systemd.tmpfiles.settings."10-rke2" = {
"/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {
"L+".argument = "${coreImages}";
};
"/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {
"L+".argument = "${canalImages}";
};
"/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
"L+".argument = "${helloImage}";
};
# Copy the canal config so that rke2 can write the remaining default values to it
"/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
"C".argument = "${canalConfig}";
};
};
# Canal CNI with VXLAN
networking.firewall.allowedUDPPorts = [ 8472 ];
networking.firewall.allowedTCPPorts = [
@@ -134,6 +92,41 @@ import ../make-test-python.nix (
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
canal-config.content = canalConfig;
# A daemonset that responds 'hello' on port 8000
network-test.content = {
apiVersion = "apps/v1";
kind = "DaemonSet";
metadata = {
name = "test";
labels.name = "test";
};
spec = {
selector.matchLabels.name = "test";
template = {
metadata.labels.name = "test";
spec.containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
imagePullPolicy = "Never";
command = [
"socat"
"TCP4-LISTEN:8000,fork"
"EXEC:echo hello"
];
}
];
};
};
};
};
};
};
@@ -145,22 +138,6 @@ import ../make-test-python.nix (
...
}:
{
# Setup image archives to be imported by rke2
systemd.tmpfiles.settings."10-rke2" = {
"/var/lib/rancher/rke2/agent/images/rke2-images-core.linux-amd64.tar.zst" = {
"L+".argument = "${coreImages}";
};
"/var/lib/rancher/rke2/agent/images/rke2-images-canal.linux-amd64.tar.zst" = {
"L+".argument = "${canalImages}";
};
"/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
"L+".argument = "${helloImage}";
};
"/var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml" = {
"C".argument = "${canalConfig}";
};
};
# Canal CNI health checks
networking.firewall.allowedTCPPorts = [ 9099 ];
# Canal CNI with VXLAN
@@ -177,6 +154,12 @@ import ../make-test-python.nix (
tokenFile = agentTokenFile;
serverAddr = "https://${nodes.server.networking.primaryIPAddress}:9345";
nodeIP = config.networking.primaryIPAddress;
manifests.canal-config.content = canalConfig;
images = [
coreImages
canalImages
helloImage
];
};
};
};
@@ -199,8 +182,7 @@ import ../make-test-python.nix (
server.succeed("${kubectl} cluster-info")
server.wait_until_succeeds("${kubectl} get serviceaccount default")
# Now create a pod on each node via a daemonset and verify they can talk to each other.
server.succeed("${kubectl} apply -f ${networkTestDaemonset}")
# Now verify that each daemonset pod can talk to each other.
server.wait_until_succeeds(
f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]'
)
@@ -217,9 +199,9 @@ import ../make-test-python.nix (
server.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
agent.wait_until_succeeds(f"ping -c 1 {pod_ip}", timeout=5)
# Verify the server can exec into the pod
# for pod in pods:
# resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -")
# assert resp.strip() == "hello", f"Unexpected response from hello daemonset: {resp.strip()}"
for pod in pods:
resp = server.succeed(f"${kubectl} exec {pod} -- socat TCP:{pod_ip}:8000 -").strip()
assert resp == "hello", f"Unexpected response from hello daemonset: {resp}"
'';
}
)

96
nixos/tests/rke2/single-node.nix
View File

@@ -26,19 +26,13 @@ import ../make-test-python.nix (
copyToRoot = pkgs.hello;
config.Entrypoint = [ "${pkgs.hello}/bin/hello" ];
};
testJobYaml = pkgs.writeText "test.yaml" ''
apiVersion: batch/v1
kind: Job
metadata:
name: test
spec:
template:
spec:
containers:
- name: test
image: "test.local/hello:local"
restartPolicy: Never
'';
# A ConfigMap in regular yaml format
cmFile = (pkgs.formats.yaml { }).generate "rke2-manifest-from-file.yaml" {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "from-file";
data.username = "foo-file";
};
in
{
name = "${rke2.name}-single-node";
@@ -51,19 +45,6 @@ import ../make-test-python.nix (
...
}:
{
# Setup image archives to be imported by rke2
systemd.tmpfiles.settings."10-rke2" = {
"/var/lib/rancher/rke2/agent/images/rke2-images-core.tar.zst" = {
"L+".argument = "${coreImages}";
};
"/var/lib/rancher/rke2/agent/images/rke2-images-canal.tar.zst" = {
"L+".argument = "${canalImages}";
};
"/var/lib/rancher/rke2/agent/images/hello.tar.zst" = {
"L+".argument = "${helloImage}";
};
};
# RKE2 needs more resources than the default
virtualisation.cores = 4;
virtualisation.memorySize = 4096;
@@ -84,6 +65,47 @@ import ../make-test-python.nix (
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
images = [
coreImages
canalImages
helloImage
];
manifests = {
test-job.content = {
apiVersion = "batch/v1";
kind = "Job";
metadata.name = "test";
spec.template.spec = {
containers = [
{
name = "hello";
image = "${helloImage.imageName}:${helloImage.imageTag}";
}
];
restartPolicy = "Never";
};
};
disabled = {
enable = false;
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "disabled";
data.username = "foo";
};
};
from-file.source = "${cmFile}";
custom-target = {
enable = true;
target = "my-manifest.json";
content = {
apiVersion = "v1";
kind = "ConfigMap";
metadata.name = "custom-target";
data.username = "foo-custom";
};
};
};
};
};
@@ -95,14 +117,28 @@ import ../make-test-python.nix (
''
start_all()
with subtest("Start cluster"):
machine.wait_for_unit("rke2-server")
machine.succeed("${kubectl} cluster-info")
machine.wait_until_succeeds("${kubectl} get serviceaccount default")
machine.succeed("${kubectl} apply -f ${testJobYaml}")
with subtest("Test job completes successfully"):
machine.wait_until_succeeds("${kubectl} wait --for 'condition=complete' job/test")
output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test")
assert output.rstrip() == "Hello, world!", f"unexpected output of test job: {output}"
output = machine.succeed("${kubectl} logs -l batch.kubernetes.io/job-name=test").rstrip()
assert output == "Hello, world!", f"unexpected output of test job: {output}"
with subtest("ConfigMap from-file exists"):
output = machine.succeed("${kubectl} get cm from-file -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-file", f"Unexpected data in Configmap from-file: {output}"
with subtest("ConfigMap custom-target exists"):
# Check that the file exists at the custom target path
machine.succeed("ls /var/lib/rancher/rke2/server/manifests/my-manifest.json")
output = machine.succeed("${kubectl} get cm custom-target -o=jsonpath='{.data.username}'").rstrip()
assert output == "foo-custom", f"Unexpected data in Configmap custom-target: {output}"
with subtest("Disabled ConfigMap doesn't exist"):
machine.fail("${kubectl} get cm disabled")
'';
}
)

2
pkgs/applications/networking/cluster/k3s/docs/ONBOARDING_MAINTAINER.md
View File

@@ -8,7 +8,7 @@ A K3s maintainer, maintains K3s's:
- [issues](https://github.com/NixOS/nixpkgs/issues?q=is%3Aissue+is%3Aopen+k3s)
- [pull requests](https://github.com/NixOS/nixpkgs/pulls?q=is%3Aopen+is%3Apr+label%3A%226.topic%3A+k3s%22)
- [NixOS tests](https://github.com/NixOS/nixpkgs/tree/master/nixos/tests/k3s)
- [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/k3s/default.nix)
- [NixOS service module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/cluster/rancher)
- [update script](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/cluster/k3s/update-script.sh) (the process of updating)
- updates (the act of updating) and [r-ryantm bot logs](https://r.ryantm.com/log/k3s/)
- deprecations

1
pkgs/applications/networking/cluster/rke2/builder.nix
View File

@@ -157,6 +157,7 @@ buildGoModule (finalAttrs: {
changelog = "https://github.com/rancher/rke2/releases/tag/v${finalAttrs.version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [
azey7f
rorosen
zimbatm
zygot