nixos/k3s: add autoDeployCharts option and use systemd-tmpfiles for content activation (#374017)
This commit is contained in:
Marcus Ramberg
@@ -597,6 +597,8 @@
|
||||
- New options for the declarative configuration of the user space part of ALSA have been introduced under [hardware.alsa](options.html#opt-hardware.alsa.enable), including setting the default capture and playback device, defining sound card aliases and volume controls.
|
||||
Note: these are intended for users not running a sound server like PulseAudio or PipeWire, but having ALSA as their only sound system.
|
||||
|
||||
- `services.k3s` now provides the `autoDeployCharts` option that allows to automatically deploy Helm charts via the k3s Helm controller.
|
||||
|
||||
- Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -20,110 +20,355 @@ let
|
||||
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
||||
imageDir = "/var/lib/rancher/k3s/agent/images";
|
||||
containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
|
||||
yamlFormat = pkgs.formats.yaml { };
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
# Manifests need a valid YAML suffix to be respected by k3s
|
||||
mkManifestTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
# Produces a list containing all duplicate manifest names
|
||||
duplicateManifests =
|
||||
with builtins;
|
||||
lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
|
||||
# Produces a list containing all duplicate chart names
|
||||
duplicateCharts =
|
||||
with builtins;
|
||||
lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);
|
||||
|
||||
manifestModule =
|
||||
let
|
||||
mkTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
in
|
||||
lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = lib.literalExpression "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkYaml docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
mkYaml name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
# Converts YAML -> JSON -> Nix
|
||||
fromYaml =
|
||||
path:
|
||||
with builtins;
|
||||
fromJSON (
|
||||
readFile (
|
||||
pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
|
||||
yq --no-colors --output-format json ${path} > $out
|
||||
''
|
||||
)
|
||||
);
|
||||
|
||||
enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests);
|
||||
linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
|
||||
linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
|
||||
linkChartEntry =
|
||||
let
|
||||
mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
in
|
||||
# Replace characters that are problematic in file names
|
||||
cleanHelmChartName =
|
||||
lib.replaceStrings
|
||||
[
|
||||
"/"
|
||||
":"
|
||||
]
|
||||
[
|
||||
"-"
|
||||
"-"
|
||||
];
|
||||
|
||||
# Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
|
||||
fetchHelm =
|
||||
{
|
||||
name,
|
||||
repo,
|
||||
version,
|
||||
hash ? lib.fakeHash,
|
||||
}:
|
||||
pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz")
|
||||
{
|
||||
inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
|
||||
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
|
||||
nativeBuildInputs = with pkgs; [
|
||||
kubernetes-helm
|
||||
cacert
|
||||
];
|
||||
}
|
||||
''
|
||||
export HOME="$PWD"
|
||||
helm repo add repository ${repo}
|
||||
helm pull repository/${name} --version ${version}
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
|
||||
# Returns the path to a YAML manifest file
|
||||
mkExtraDeployManifest =
|
||||
x:
|
||||
# x is a derivation that provides a YAML file
|
||||
if lib.isDerivation x then
|
||||
x.outPath
|
||||
# x is an attribute set that needs to be converted to a YAML file
|
||||
else if builtins.isAttrs x then
|
||||
(yamlFormat.generate "extra-deploy-chart-manifest" x)
|
||||
# assume x is a path to a YAML file
|
||||
else
|
||||
x;
|
||||
|
||||
# Generate a HelmChart custom resource.
|
||||
mkHelmChartCR =
|
||||
name: value:
|
||||
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
|
||||
let
|
||||
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
|
||||
# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
|
||||
valuesContent = builtins.toJSON chartValues;
|
||||
in
|
||||
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
|
||||
# values
|
||||
lib.recursiveUpdate {
|
||||
apiVersion = "helm.cattle.io/v1";
|
||||
kind = "HelmChart";
|
||||
metadata = {
|
||||
inherit name;
|
||||
namespace = "kube-system";
|
||||
};
|
||||
spec = {
|
||||
inherit valuesContent;
|
||||
inherit (value) targetNamespace createNamespace;
|
||||
chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
|
||||
};
|
||||
} value.extraFieldDefinitions;
|
||||
|
||||
activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
|
||||
${lib.optionalString (
|
||||
builtins.length enabledManifests > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
|
||||
${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
||||
${lib.optionalString (
|
||||
builtins.length cfg.images > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
|
||||
# Generate a HelmChart custom resource together with extraDeploy manifests. This
|
||||
# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
|
||||
# deploys.
|
||||
mkAutoDeployChartManifest = name: value: {
|
||||
# target is the final name of the link created for the manifest file
|
||||
target = mkManifestTarget name;
|
||||
inherit (value) enable package;
|
||||
# source is a store path containing the complete manifest file
|
||||
source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
|
||||
[
|
||||
(yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
|
||||
]
|
||||
# alternate the YAML doc seperator (---) and extraDeploy manifests to create
|
||||
# multi document YAMLs
|
||||
++ (lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkExtraDeployManifest x)
|
||||
]) value.extraDeploy)
|
||||
);
|
||||
};
|
||||
|
||||
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
|
||||
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
|
||||
${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
|
||||
autoDeployChartsModule = lib.types.submodule (
|
||||
{ config, ... }:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
example = false;
|
||||
description = ''
|
||||
Whether to enable the installation of this Helm chart. Note that setting
|
||||
this option to `false` will not uninstall the chart from the cluster, if
|
||||
it was previously installed. Please use the the `--disable` flag or `.skip`
|
||||
files to delete/disable Helm charts, as mentioned in the
|
||||
[docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
||||
'';
|
||||
};
|
||||
|
||||
${lib.optionalString (cfg.containerdConfigTemplate != null) ''
|
||||
mkdir -p $(dirname ${containerdConfigTemplateFile})
|
||||
${pkgs.coreutils-full}/bin/ln -sfn ${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate} ${containerdConfigTemplateFile}
|
||||
''}
|
||||
'';
|
||||
repo = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "https://kubernetes.github.io/ingress-nginx";
|
||||
description = ''
|
||||
The repo of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
name = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "ingress-nginx";
|
||||
description = ''
|
||||
The name of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
version = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "4.7.0";
|
||||
description = ''
|
||||
The version of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
hash = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
|
||||
description = ''
|
||||
The hash of the packaged Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = with lib.types; either path package;
|
||||
example = lib.literalExpression "../my-helm-chart.tgz";
|
||||
description = ''
|
||||
The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
|
||||
and `hash` in case of conflicts.
|
||||
'';
|
||||
};
|
||||
|
||||
targetNamespace = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
default = "default";
|
||||
example = "kube-system";
|
||||
description = "The namespace in which the Helm chart gets installed.";
|
||||
};
|
||||
|
||||
createNamespace = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = "Whether to create the target namespace if not present.";
|
||||
};
|
||||
|
||||
values = lib.mkOption {
|
||||
type = with lib.types; either path attrs;
|
||||
default = { };
|
||||
example = {
|
||||
replicaCount = 3;
|
||||
hostName = "my-host";
|
||||
server = {
|
||||
name = "nginx";
|
||||
port = 80;
|
||||
};
|
||||
};
|
||||
description = ''
|
||||
Override default chart values via Nix expressions. This is equivalent to setting
|
||||
values in a `values.yaml` file.
|
||||
|
||||
WARNING: The values (including secrets!) specified here are exposed unencrypted
|
||||
in the world-readable nix store.
|
||||
'';
|
||||
};
|
||||
|
||||
extraDeploy = lib.mkOption {
|
||||
type = with lib.types; listOf (either path attrs);
|
||||
default = [ ];
|
||||
example = lib.literalExpression ''
|
||||
[
|
||||
../manifests/my-extra-deployment.yaml
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
'';
|
||||
description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
|
||||
};
|
||||
|
||||
extraFieldDefinitions = lib.mkOption {
|
||||
inherit (yamlFormat) type;
|
||||
default = { };
|
||||
example = {
|
||||
spec = {
|
||||
bootstrap = true;
|
||||
helmVersion = "v2";
|
||||
backOffLimit = 3;
|
||||
jobImage = "custom-helm-controller:v0.0.1";
|
||||
};
|
||||
};
|
||||
description = ''
|
||||
Extra HelmChart field definitions that are merged with the rest of the HelmChart
|
||||
custom resource. This can be used to set advanced fields or to overwrite
|
||||
generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
|
||||
for possible fields.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config.package = lib.mkDefault (fetchHelm {
|
||||
inherit (config)
|
||||
repo
|
||||
name
|
||||
version
|
||||
hash
|
||||
;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
manifestModule = lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkManifestTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(yamlFormat.generate docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
yamlFormat.generate name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
);
|
||||
in
|
||||
{
|
||||
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
||||
@@ -242,78 +487,80 @@ in
|
||||
type = lib.types.attrsOf manifestModule;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
{
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
};
|
||||
};
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
'';
|
||||
description = ''
|
||||
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
||||
@@ -337,10 +584,9 @@ in
|
||||
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
||||
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
||||
The specified charts will only be placed on the file system and made available to the
|
||||
Kubernetes APIServer from within the cluster, you may use the
|
||||
[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy the charts. This option only makes sense on server nodes
|
||||
(`role = server`).
|
||||
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
|
||||
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
|
||||
'';
|
||||
};
|
||||
|
||||
@@ -450,6 +696,53 @@ in
|
||||
set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
|
||||
'';
|
||||
};
|
||||
|
||||
autoDeployCharts = lib.mkOption {
|
||||
type = lib.types.attrsOf autoDeployChartsModule;
|
||||
apply = lib.mapAttrs mkAutoDeployChartManifest;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
{
|
||||
harbor = {
|
||||
name = "harbor";
|
||||
repo = "https://helm.goharbor.io";
|
||||
version = "1.14.0";
|
||||
hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
|
||||
values = {
|
||||
existingSecretAdminPassword = "harbor-admin";
|
||||
expose = {
|
||||
tls = {
|
||||
enabled = true;
|
||||
certSource = "secret";
|
||||
secret.secretName = "my-tls-secret";
|
||||
};
|
||||
ingress = {
|
||||
hosts.core = "example.com";
|
||||
className = "nginx";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
custom-chart = {
|
||||
package = ../charts/my-chart.tgz;
|
||||
values = ../values/my-values.yaml;
|
||||
extraFieldDefinitions = {
|
||||
spec.timeout = "60s";
|
||||
};
|
||||
};
|
||||
}
|
||||
'';
|
||||
description = ''
|
||||
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
|
||||
attribute names that are also used in the [](#opt-services.k3s.manifests) and
|
||||
[](#opt-services.k3s.charts) options. Manifests with the same name will override
|
||||
auto deploying charts with the same name. Similiarly, charts with the same name will
|
||||
overwrite the Helm chart contained in auto deploying charts. This option only makes
|
||||
sense on server nodes (`role = server`). See the
|
||||
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# implementation
|
||||
@@ -462,6 +755,15 @@ in
|
||||
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
||||
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
|
||||
"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (duplicateManifests != [ ])
|
||||
"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
|
||||
)
|
||||
++ (lib.optional (duplicateCharts != [ ])
|
||||
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
|
||||
)
|
||||
++ (lib.optional (
|
||||
cfg.disableAgent && cfg.images != [ ]
|
||||
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
|
||||
@@ -486,6 +788,50 @@ in
|
||||
|
||||
environment.systemPackages = [ config.services.k3s.package ];
|
||||
|
||||
# Use systemd-tmpfiles to activate k3s content
|
||||
systemd.tmpfiles.settings."10-k3s" =
|
||||
let
|
||||
# Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
|
||||
enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
|
||||
# Merge charts with charts contained in enabled auto deploying charts
|
||||
helmCharts =
|
||||
(lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
|
||||
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
|
||||
))
|
||||
// cfg.charts;
|
||||
# Make a systemd-tmpfiles rule for a manifest
|
||||
mkManifestRule = manifest: {
|
||||
name = "${manifestDir}/${manifest.target}";
|
||||
value = {
|
||||
"L+".argument = "${manifest.source}";
|
||||
};
|
||||
};
|
||||
# Ensure that all chart targets have a .tgz suffix
|
||||
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
# Make a systemd-tmpfiles rule for a chart
|
||||
mkChartRule = target: source: {
|
||||
name = "${chartDir}/${mkChartTarget target}";
|
||||
value = {
|
||||
"L+".argument = "${source}";
|
||||
};
|
||||
};
|
||||
# Make a systemd-tmpfiles rule for a container image
|
||||
mkImageRule = image: {
|
||||
name = "${imageDir}/${image.name}";
|
||||
value = {
|
||||
"L+".argument = "${image}";
|
||||
};
|
||||
};
|
||||
in
|
||||
(lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
|
||||
// (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
|
||||
// (builtins.listToAttrs (map mkImageRule cfg.images))
|
||||
// (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
|
||||
${containerdConfigTemplateFile} = {
|
||||
"L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
|
||||
};
|
||||
});
|
||||
|
||||
systemd.services.k3s =
|
||||
let
|
||||
kubeletParams =
|
||||
@@ -533,7 +879,6 @@ in
|
||||
LimitCORE = "infinity";
|
||||
TasksMax = "infinity";
|
||||
EnvironmentFile = cfg.environmentFile;
|
||||
ExecStartPre = activateK3sContent;
|
||||
ExecStart = lib.concatStringsSep " \\\n " (
|
||||
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
|
||||
++ (lib.optional cfg.clusterInit "--cluster-init")
|
||||
|
||||
135
nixos/tests/k3s/auto-deploy-charts.nix
Normal file
135
nixos/tests/k3s/auto-deploy-charts.nix
Normal file
@@ -0,0 +1,135 @@
|
||||
# Tests whether container images are imported and auto deploying Helm charts work
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
k3s,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
testImageEnv = pkgs.buildEnv {
|
||||
name = "k3s-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
busybox
|
||||
hello
|
||||
];
|
||||
};
|
||||
testImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/test";
|
||||
tag = "local";
|
||||
# Slightly reduces the time needed to import image
|
||||
compressor = "zstd";
|
||||
copyToRoot = testImageEnv;
|
||||
};
|
||||
# pack the test helm chart as a .tgz archive
|
||||
package =
|
||||
pkgs.runCommand "k3s-test-chart.tgz"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.kubernetes-helm ];
|
||||
}
|
||||
''
|
||||
helm package ${./k3s-test-chart}
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
# The common Helm chart that is used in this test
|
||||
testChart = {
|
||||
inherit package;
|
||||
values = {
|
||||
runCommand = "hello";
|
||||
image = {
|
||||
repository = testImage.imageName;
|
||||
tag = testImage.imageTag;
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${k3s.name}-auto-deploy-helm";
|
||||
meta.maintainers = lib.teams.k3s.members;
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
# k3s uses enough resources the default vm fails.
|
||||
virtualisation = {
|
||||
memorySize = 1536;
|
||||
diskSize = 4096;
|
||||
};
|
||||
environment.systemPackages = [ pkgs.yq-go ];
|
||||
services.k3s = {
|
||||
enable = true;
|
||||
package = k3s;
|
||||
# Slightly reduce resource usage
|
||||
extraFlags = [
|
||||
"--disable coredns"
|
||||
"--disable local-storage"
|
||||
"--disable metrics-server"
|
||||
"--disable servicelb"
|
||||
"--disable traefik"
|
||||
];
|
||||
images = [
|
||||
# Provides the k3s Helm controller
|
||||
k3s.airgapImages
|
||||
testImage
|
||||
];
|
||||
autoDeployCharts = {
|
||||
# regular test chart that should get installed
|
||||
hello = testChart;
|
||||
# disabled chart that should not get installed
|
||||
disabled = testChart // {
|
||||
enable = false;
|
||||
};
|
||||
# advanced chart that should get installed in the "test" namespace with a custom
|
||||
# timeout and overridden values
|
||||
advanced = testChart // {
|
||||
# create the "test" namespace via extraDeploy for testing
|
||||
extraDeploy = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "test";
|
||||
}
|
||||
];
|
||||
extraFieldDefinitions = {
|
||||
spec = {
|
||||
# overwrite chart values
|
||||
valuesContent = ''
|
||||
runCommand: "echo 'advanced hello'"
|
||||
image:
|
||||
repository: ${testImage.imageName}
|
||||
tag: ${testImage.imageTag}
|
||||
'';
|
||||
# overwrite the chart namespace
|
||||
targetNamespace = "test";
|
||||
# set a custom timeout
|
||||
timeout = "69s";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
import json
|
||||
|
||||
machine.wait_for_unit("k3s")
|
||||
# check existence/absence of chart manifest files
|
||||
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
|
||||
machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
|
||||
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
|
||||
# check that the timeout is set correctly, select only the first doc in advanced.yaml
|
||||
advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
|
||||
assert advancedManifest["spec"]["timeout"] == "69s", f"unexpected value for spec.timeout: {advancedManifest["spec"]["timeout"]}"
|
||||
# wait for test jobs to complete
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
|
||||
# check output of test jobs
|
||||
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
|
||||
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
|
||||
# strip the output to remove trailing whitespaces
|
||||
assert hello_output.rstrip() == "Hello, world!", f"unexpected output of hello job: {hello_output}"
|
||||
assert advanced_output.rstrip() == "advanced hello", f"unexpected output of advanced job: {advanced_output}"
|
||||
'';
|
||||
}
|
||||
)
|
||||
@@ -11,6 +11,9 @@ in
|
||||
_: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
|
||||
auto-deploy-charts = lib.mapAttrs (
|
||||
_: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
containerd-config = lib.mapAttrs (
|
||||
_: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
|
||||
24
nixos/tests/k3s/k3s-test-chart/Chart.yaml
Normal file
24
nixos/tests/k3s/k3s-test-chart/Chart.yaml
Normal file
@@ -0,0 +1,24 @@
|
||||
apiVersion: v2
|
||||
name: k3s-test-chart
|
||||
description: A Helm chart that is used in k3s NixOS tests.
|
||||
|
||||
# A chart can be either an 'application' or a 'library' chart.
|
||||
#
|
||||
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||
# to be deployed.
|
||||
#
|
||||
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||
type: application
|
||||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.1.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "1.16.0"
|
||||
14
nixos/tests/k3s/k3s-test-chart/templates/job.yaml
Normal file
14
nixos/tests/k3s/k3s-test-chart/templates/job.yaml
Normal file
@@ -0,0 +1,14 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ .Release.Name | quote }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
command: ["sh"]
|
||||
args: ["-c", "{{ .Values.runCommand }}"]
|
||||
restartPolicy: {{ .Values.restartPolicy | quote }}
|
||||
5
nixos/tests/k3s/k3s-test-chart/values.yaml
Normal file
5
nixos/tests/k3s/k3s-test-chart/values.yaml
Normal file
@@ -0,0 +1,5 @@
|
||||
restartPolicy: "Never"
|
||||
runCommand: ""
|
||||
image:
|
||||
repository: foo
|
||||
tag: 1.0.0
|
||||
Reference in New Issue
Block a user