From 95b894bad77bf210e83cb7a38e7130409cefe1fe Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Wed, 15 Jan 2025 08:28:08 +0100 Subject: [PATCH 01/23] nixos/k3s: add `autoDeployCharts` option The `autoDeployCharts` option further improves the auto deploying capabilities of the k3s module by allowing to deploy and configure Helm charts that are then instaled via the k3s Helm controller. Although this was also previously possible by using auto deploying manifests, it required some knowledge of the k3s Helm controller and led to a lot of boilerplate code. --- .../manual/release-notes/rl-2505.section.md | 2 + .../modules/services/cluster/k3s/default.nix | 636 ++++++++++++++---- nixos/tests/k3s/auto-deploy-charts.nix | 135 ++++ nixos/tests/k3s/default.nix | 3 + nixos/tests/k3s/k3s-test-chart/Chart.yaml | 24 + .../k3s/k3s-test-chart/templates/job.yaml | 14 + nixos/tests/k3s/k3s-test-chart/values.yaml | 5 + 7 files changed, 671 insertions(+), 148 deletions(-) create mode 100644 nixos/tests/k3s/auto-deploy-charts.nix create mode 100644 nixos/tests/k3s/k3s-test-chart/Chart.yaml create mode 100644 nixos/tests/k3s/k3s-test-chart/templates/job.yaml create mode 100644 nixos/tests/k3s/k3s-test-chart/values.yaml diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index a6f796885f96..69a0be99b812 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -392,6 +392,8 @@ - New options for the declarative configuration of the user space part of ALSA have been introduced under [hardware.alsa](options.html#opt-hardware.alsa.enable), including setting the default capture and playback device, defining sound card aliases and volume controls. Note: these are intended for users not running a sound server like PulseAudio or PipeWire, but having ALSA as their only sound system. +- `services.k3s` now provides the `autoDeployCharts` option that allows to automatically deploy Helm charts via the k3s Helm controller. + - Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added. Example: diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix index 97019ba5cb4d..2d182856ab5a 100644 --- a/nixos/modules/services/cluster/k3s/default.nix +++ b/nixos/modules/services/cluster/k3s/default.nix @@ -20,103 +20,386 @@ let chartDir = "/var/lib/rancher/k3s/server/static/charts"; imageDir = "/var/lib/rancher/k3s/agent/images"; containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"; + yamlFormat = pkgs.formats.yaml { }; + yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n"; + # Manifests need a valid YAML suffix to be respected by k3s + mkManifestTarget = + name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml"; + # Produces a list containing all duplicate manifest names + duplicateManifests = + with builtins; + lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests); + # Produces a list containing all duplicate chart names + duplicateCharts = + with builtins; + lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts); - manifestModule = - let - mkTarget = - name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml"; - in - lib.types.submodule ( - { - name, - config, - options, - ... - }: - { - options = { - enable = lib.mkOption { - type = lib.types.bool; - default = true; - description = "Whether this manifest file should be generated."; - }; - - target = lib.mkOption { - type = lib.types.nonEmptyStr; - example = lib.literalExpression "manifest.yaml"; - description = '' - Name of the symlink (relative to {file}`${manifestDir}`). - Defaults to the attribute name. - ''; - }; - - content = lib.mkOption { - type = with lib.types; nullOr (either attrs (listOf attrs)); - default = null; - description = '' - Content of the manifest file. A single attribute set will - generate a single document YAML file. A list of attribute sets - will generate multiple documents separated by `---` in a single - YAML file. - ''; - }; - - source = lib.mkOption { - type = lib.types.path; - example = lib.literalExpression "./manifests/app.yaml"; - description = '' - Path of the source `.yaml` file. - ''; - }; - }; - - config = { - target = lib.mkDefault (mkTarget name); - source = lib.mkIf (config.content != null) ( - let - name' = "k3s-manifest-" + builtins.baseNameOf name; - docName = "k3s-manifest-doc-" + builtins.baseNameOf name; - yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n"; - mkYaml = name: x: (pkgs.formats.yaml { }).generate name x; - mkSource = - value: - if builtins.isList value then - pkgs.concatText name' ( - lib.concatMap (x: [ - yamlDocSeparator - (mkYaml docName x) - ]) value - ) - else - mkYaml name' value; - in - lib.mkDerivedConfig options.content mkSource - ); - }; - } + # Converts YAML -> JSON -> Nix + fromYaml = + path: + with builtins; + fromJSON ( + readFile ( + pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } '' + yq --no-colors --output-format json ${path} > $out + '' + ) ); + # Replace characters that are problematic in file names + cleanHelmChartName = + lib.replaceStrings + [ + "/" + ":" + ] + [ + "-" + "-" + ]; + + # Fetch a Helm chart from a public registry. This only supports a basic Helm pull. + fetchHelm = + { + name, + repo, + version, + hash ? lib.fakeHash, + }: + pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz") + { + inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + nativeBuildInputs = with pkgs; [ + kubernetes-helm + cacert + ]; + } + '' + export HOME="$PWD" + helm repo add repository ${repo} + helm pull repository/${name} --version ${version} + mv ./*.tgz $out + ''; + + # Returns the path to a YAML manifest file + mkExtraDeployManifest = + x: + # x is a derivation that provides a YAML file + if lib.isDerivation x then + x.outPath + # x is an attribute set that needs to be converted to a YAML file + else if builtins.isAttrs x then + (yamlFormat.generate "extra-deploy-chart-manifest" x) + # assume x is a path to a YAML file + else + x; + + # Generate a HelmChart custom resource. + mkHelmChartCR = + name: value: + let + chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values; + # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller + valuesContent = builtins.toJSON chartValues; + in + # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated + # values + lib.recursiveUpdate { + apiVersion = "helm.cattle.io/v1"; + kind = "HelmChart"; + metadata = { + inherit name; + namespace = "kube-system"; + }; + spec = { + inherit valuesContent; + inherit (value) targetNamespace createNamespace; + chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz"; + }; + } value.extraFieldDefinitions; + + # Generate a HelmChart custom resource together with extraDeploy manifests. This + # generates possibly a multi document YAML file that the auto deploy mechanism of k3s + # deploys. + mkAutoDeployChartManifest = name: value: { + # target is the final name of the link created for the manifest file + target = mkManifestTarget name; + inherit (value) enable package; + # source is a store path containing the complete manifest file + source = pkgs.concatText "auto-deploy-chart-${name}.yaml" ( + [ + (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value)) + ] + # alternate the YAML doc seperator (---) and extraDeploy manifests to create + # multi document YAMLs + ++ (lib.concatMap (x: [ + yamlDocSeparator + (mkExtraDeployManifest x) + ]) value.extraDeploy) + ); + }; + + autoDeployChartsModule = lib.types.submodule ( + { config, ... }: + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + example = false; + description = '' + Whether to enable the installation of this Helm chart. Note that setting + this option to `false` will not uninstall the chart from the cluster, if + it was previously installed. Please use the the `--disable` flag or `.skip` + files to delete/disable Helm charts, as mentioned in the + [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests). + ''; + }; + + repo = lib.mkOption { + type = lib.types.nonEmptyStr; + example = "https://kubernetes.github.io/ingress-nginx"; + description = '' + The repo of the Helm chart. Only has an effect if `package` is not set. + The Helm chart is fetched during build time and placed as a `.tgz` archive on the + filesystem. + ''; + }; + + name = lib.mkOption { + type = lib.types.nonEmptyStr; + example = "ingress-nginx"; + description = '' + The name of the Helm chart. Only has an effect if `package` is not set. + The Helm chart is fetched during build time and placed as a `.tgz` archive on the + filesystem. + ''; + }; + + version = lib.mkOption { + type = lib.types.nonEmptyStr; + example = "4.7.0"; + description = '' + The version of the Helm chart. Only has an effect if `package` is not set. + The Helm chart is fetched during build time and placed as a `.tgz` archive on the + filesystem. + ''; + }; + + hash = lib.mkOption { + type = lib.types.str; + example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA="; + description = '' + The hash of the packaged Helm chart. Only has an effect if `package` is not set. + The Helm chart is fetched during build time and placed as a `.tgz` archive on the + filesystem. + ''; + }; + + package = lib.mkOption { + type = with lib.types; either path package; + example = lib.literalExpression "../my-helm-chart.tgz"; + description = '' + The packaged Helm chart. Overwrites the options `repo`, `name`, `version` + and `hash` in case of conflicts. + ''; + }; + + targetNamespace = lib.mkOption { + type = lib.types.nonEmptyStr; + default = "default"; + example = "kube-system"; + description = "The namespace in which the Helm chart gets installed."; + }; + + createNamespace = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether to create the target namespace if not present."; + }; + + values = lib.mkOption { + type = with lib.types; either path attrs; + default = { }; + example = { + replicaCount = 3; + hostName = "my-host"; + server = { + name = "nginx"; + port = 80; + }; + }; + description = '' + Override default chart values via Nix expressions. This is equivalent to setting + values in a `values.yaml` file. + + WARNING: The values (including secrets!) specified here are exposed unencrypted + in the world-readable nix store. + ''; + }; + + extraDeploy = lib.mkOption { + type = with lib.types; listOf (either path attrs); + default = [ ]; + example = lib.literalExpression '' + [ + ../manifests/my-extra-deployment.yaml + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "app-service"; + }; + spec = { + selector = { + "app.kubernetes.io/name" = "MyApp"; + }; + ports = [ + { + name = "name-of-service-port"; + protocol = "TCP"; + port = 80; + targetPort = "http-web-svc"; + } + ]; + }; + } + ]; + ''; + description = "List of extra Kubernetes manifests to deploy with this Helm chart."; + }; + + extraFieldDefinitions = lib.mkOption { + inherit (yamlFormat) type; + default = { }; + example = { + spec = { + bootstrap = true; + helmVersion = "v2"; + backOffLimit = 3; + jobImage = "custom-helm-controller:v0.0.1"; + }; + }; + description = '' + Extra HelmChart field definitions that are merged with the rest of the HelmChart + custom resource. This can be used to set advanced fields or to overwrite + generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions + for possible fields. + ''; + }; + }; + + config.package = lib.mkDefault (fetchHelm { + inherit (config) + repo + name + version + hash + ; + }); + } + ); + + manifestModule = lib.types.submodule ( + { + name, + config, + options, + ... + }: + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether this manifest file should be generated."; + }; + + target = lib.mkOption { + type = lib.types.nonEmptyStr; + example = "manifest.yaml"; + description = '' + Name of the symlink (relative to {file}`${manifestDir}`). + Defaults to the attribute name. + ''; + }; + + content = lib.mkOption { + type = with lib.types; nullOr (either attrs (listOf attrs)); + default = null; + description = '' + Content of the manifest file. A single attribute set will + generate a single document YAML file. A list of attribute sets + will generate multiple documents separated by `---` in a single + YAML file. + ''; + }; + + source = lib.mkOption { + type = lib.types.path; + example = lib.literalExpression "./manifests/app.yaml"; + description = '' + Path of the source `.yaml` file. + ''; + }; + }; + + config = { + target = lib.mkDefault (mkManifestTarget name); + source = lib.mkIf (config.content != null) ( + let + name' = "k3s-manifest-" + builtins.baseNameOf name; + docName = "k3s-manifest-doc-" + builtins.baseNameOf name; + mkSource = + value: + if builtins.isList value then + pkgs.concatText name' ( + lib.concatMap (x: [ + yamlDocSeparator + (yamlFormat.generate docName x) + ]) value + ) + else + yamlFormat.generate name' value; + in + lib.mkDerivedConfig options.content mkSource + ); + }; + } + ); + + # TODO: use tmpfiles enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests); + enabledHelmManifests = lib.filter (m: m.enable) (lib.attrValues cfg.autoDeployCharts); + enabledAutoDeployCharts = lib.concatMapAttrs (n: v: { ${n} = v.package; }) ( + lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts + ); linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}"; linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}"; linkChartEntry = let - mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz"; + mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz"; in name: value: - "${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}"; + "${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkChartTarget (builtins.baseNameOf name)}"; activateK3sContent = pkgs.writeShellScript "activate-k3s-content" '' ${lib.optionalString ( - builtins.length enabledManifests > 0 + builtins.length (enabledManifests ++ enabledHelmManifests) > 0 ) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"} - ${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"} + ${lib.optionalString ( + cfg.charts != { } || enabledAutoDeployCharts != { } + ) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"} ${lib.optionalString ( builtins.length cfg.images > 0 ) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"} ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)} + ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledHelmManifests)} ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)} + ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry enabledAutoDeployCharts)} ${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)} ${lib.optionalString (cfg.containerdConfigTemplate != null) '' @@ -242,78 +525,80 @@ in type = lib.types.attrsOf manifestModule; default = { }; example = lib.literalExpression '' - deployment.source = ../manifests/deployment.yaml; - my-service = { - enable = false; - target = "app-service.yaml"; - content = { - apiVersion = "v1"; - kind = "Service"; - metadata = { - name = "app-service"; - }; - spec = { - selector = { - "app.kubernetes.io/name" = "MyApp"; + { + deployment.source = ../manifests/deployment.yaml; + my-service = { + enable = false; + target = "app-service.yaml"; + content = { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "app-service"; + }; + spec = { + selector = { + "app.kubernetes.io/name" = "MyApp"; + }; + ports = [ + { + name = "name-of-service-port"; + protocol = "TCP"; + port = 80; + targetPort = "http-web-svc"; + } + ]; }; - ports = [ - { - name = "name-of-service-port"; - protocol = "TCP"; - port = 80; - targetPort = "http-web-svc"; - } - ]; }; - } - }; + }; - nginx.content = [ - { - apiVersion = "v1"; - kind = "Pod"; - metadata = { - name = "nginx"; - labels = { - "app.kubernetes.io/name" = "MyApp"; + nginx.content = [ + { + apiVersion = "v1"; + kind = "Pod"; + metadata = { + name = "nginx"; + labels = { + "app.kubernetes.io/name" = "MyApp"; + }; }; - }; - spec = { - containers = [ - { - name = "nginx"; - image = "nginx:1.14.2"; - ports = [ - { - containerPort = 80; - name = "http-web-svc"; - } - ]; - } - ]; - }; - } - { - apiVersion = "v1"; - kind = "Service"; - metadata = { - name = "nginx-service"; - }; - spec = { - selector = { - "app.kubernetes.io/name" = "MyApp"; + spec = { + containers = [ + { + name = "nginx"; + image = "nginx:1.14.2"; + ports = [ + { + containerPort = 80; + name = "http-web-svc"; + } + ]; + } + ]; }; - ports = [ - { - name = "name-of-service-port"; - protocol = "TCP"; - port = 80; - targetPort = "http-web-svc"; - } - ]; - }; - } - ]; + } + { + apiVersion = "v1"; + kind = "Service"; + metadata = { + name = "nginx-service"; + }; + spec = { + selector = { + "app.kubernetes.io/name" = "MyApp"; + }; + ports = [ + { + name = "name-of-service-port"; + protocol = "TCP"; + port = 80; + targetPort = "http-web-svc"; + } + ]; + }; + } + ]; + }; ''; description = '' Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts. @@ -337,10 +622,9 @@ in Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts. The attribute name will be used as the link target (relative to {file}`${chartDir}`). The specified charts will only be placed on the file system and made available to the - Kubernetes APIServer from within the cluster, you may use the - [k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller) - to deploy the charts. This option only makes sense on server nodes - (`role = server`). + Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts) + option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller) + to deploy Helm charts. This option only makes sense on server nodes (`role = server`). ''; }; @@ -450,6 +734,53 @@ in set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`. ''; }; + + autoDeployCharts = lib.mkOption { + type = lib.types.attrsOf autoDeployChartsModule; + apply = lib.mapAttrs mkAutoDeployChartManifest; + default = { }; + example = lib.literalExpression '' + { + harbor = { + name = "harbor"; + repo = "https://helm.goharbor.io"; + version = "1.14.0"; + hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU="; + values = { + existingSecretAdminPassword = "harbor-admin"; + expose = { + tls = { + enabled = true; + certSource = "secret"; + secret.secretName = "my-tls-secret"; + }; + ingress = { + hosts.core = "example.com"; + className = "nginx"; + }; + }; + }; + }; + + custom-chart = { + package = ../charts/my-chart.tgz; + values = ../values/my-values.yaml; + extraFieldDefinitions = { + spec.timeout = "60s"; + }; + }; + } + ''; + description = '' + Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use + attribute names that are also used in the [](#opt-services.k3s.manifests) and + [](#opt-services.k3s.charts) options. Manifests with the same name will override + auto deploying charts with the same name. Similiarly, charts with the same name will + overwrite the Helm chart contained in auto deploying charts. This option only makes + sense on server nodes (`role = server`). See the + [k3s Helm documentation](https://docs.k3s.io/helm) for further information. + ''; + }; }; # implementation @@ -462,6 +793,15 @@ in ++ (lib.optional (cfg.role != "server" && cfg.charts != { }) "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node." ) + ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { }) + "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node." + ) + ++ (lib.optional (duplicateManifests != [ ]) + "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}." + ) + ++ (lib.optional (duplicateCharts != [ ]) + "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}." + ) ++ (lib.optional ( cfg.disableAgent && cfg.images != [ ] ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node") diff --git a/nixos/tests/k3s/auto-deploy-charts.nix b/nixos/tests/k3s/auto-deploy-charts.nix new file mode 100644 index 000000000000..f64728f44f64 --- /dev/null +++ b/nixos/tests/k3s/auto-deploy-charts.nix @@ -0,0 +1,135 @@ +# Tests whether container images are imported and auto deploying Helm charts work +import ../make-test-python.nix ( + { + k3s, + lib, + pkgs, + ... + }: + let + testImageEnv = pkgs.buildEnv { + name = "k3s-pause-image-env"; + paths = with pkgs; [ + busybox + hello + ]; + }; + testImage = pkgs.dockerTools.buildImage { + name = "test.local/test"; + tag = "local"; + # Slightly reduces the time needed to import image + compressor = "zstd"; + copyToRoot = testImageEnv; + }; + # pack the test helm chart as a .tgz archive + package = + pkgs.runCommand "k3s-test-chart.tgz" + { + nativeBuildInputs = [ pkgs.kubernetes-helm ]; + } + '' + helm package ${./k3s-test-chart} + mv ./*.tgz $out + ''; + # The common Helm chart that is used in this test + testChart = { + inherit package; + values = { + runCommand = "hello"; + image = { + repository = testImage.imageName; + tag = testImage.imageTag; + }; + }; + }; + in + { + name = "${k3s.name}-auto-deploy-helm"; + meta.maintainers = lib.teams.k3s.members; + nodes.machine = + { pkgs, ... }: + { + # k3s uses enough resources the default vm fails. + virtualisation = { + memorySize = 1536; + diskSize = 4096; + }; + environment.systemPackages = [ pkgs.yq-go ]; + services.k3s = { + enable = true; + package = k3s; + # Slightly reduce resource usage + extraFlags = [ + "--disable coredns" + "--disable local-storage" + "--disable metrics-server" + "--disable servicelb" + "--disable traefik" + ]; + images = [ + # Provides the k3s Helm controller + k3s.airgapImages + testImage + ]; + autoDeployCharts = { + # regular test chart that should get installed + hello = testChart; + # disabled chart that should not get installed + disabled = testChart // { + enable = false; + }; + # advanced chart that should get installed in the "test" namespace with a custom + # timeout and overridden values + advanced = testChart // { + # create the "test" namespace via extraDeploy for testing + extraDeploy = [ + { + apiVersion = "v1"; + kind = "Namespace"; + metadata.name = "test"; + } + ]; + extraFieldDefinitions = { + spec = { + # overwrite chart values + valuesContent = '' + runCommand: "echo 'advanced hello'" + image: + repository: ${testImage.imageName} + tag: ${testImage.imageTag} + ''; + # overwrite the chart namespace + targetNamespace = "test"; + # set a custom timeout + timeout = "69s"; + }; + }; + }; + }; + }; + }; + + testScript = # python + '' + import json + + machine.wait_for_unit("k3s") + # check existence/absence of chart manifest files + machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml") + machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml") + machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml") + # check that the timeout is set correctly, select only the first doc in advanced.yaml + advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml")) + assert advancedManifest["spec"]["timeout"] == "69s", f"unexpected value for spec.timeout: {advancedManifest["spec"]["timeout"]}" + # wait for test jobs to complete + machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180) + machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180) + # check output of test jobs + hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello") + advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced") + # strip the output to remove trailing whitespaces + assert hello_output.rstrip() == "Hello, world!", f"unexpected output of hello job: {hello_output}" + assert advanced_output.rstrip() == "advanced hello", f"unexpected output of advanced job: {advanced_output}" + ''; + } +) diff --git a/nixos/tests/k3s/default.nix b/nixos/tests/k3s/default.nix index 7edaf6f38ed2..4ee3cb760b39 100644 --- a/nixos/tests/k3s/default.nix +++ b/nixos/tests/k3s/default.nix @@ -11,6 +11,9 @@ in _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; } ) allK3s; auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s; + auto-deploy-charts = lib.mapAttrs ( + _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; } + ) allK3s; containerd-config = lib.mapAttrs ( _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; } ) allK3s; diff --git a/nixos/tests/k3s/k3s-test-chart/Chart.yaml b/nixos/tests/k3s/k3s-test-chart/Chart.yaml new file mode 100644 index 000000000000..9cc0ae87678f --- /dev/null +++ b/nixos/tests/k3s/k3s-test-chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: k3s-test-chart +description: A Helm chart that is used in k3s NixOS tests. + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/nixos/tests/k3s/k3s-test-chart/templates/job.yaml b/nixos/tests/k3s/k3s-test-chart/templates/job.yaml new file mode 100644 index 000000000000..029453e56219 --- /dev/null +++ b/nixos/tests/k3s/k3s-test-chart/templates/job.yaml @@ -0,0 +1,14 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name | quote }} + namespace: {{ .Release.Namespace | quote }} +spec: + template: + spec: + containers: + - name: test + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["sh"] + args: ["-c", "{{ .Values.runCommand }}"] + restartPolicy: {{ .Values.restartPolicy | quote }} diff --git a/nixos/tests/k3s/k3s-test-chart/values.yaml b/nixos/tests/k3s/k3s-test-chart/values.yaml new file mode 100644 index 000000000000..cca4a557fd90 --- /dev/null +++ b/nixos/tests/k3s/k3s-test-chart/values.yaml @@ -0,0 +1,5 @@ +restartPolicy: "Never" +runCommand: "" +image: + repository: foo + tag: 1.0.0 From d3cd8299b44e829500c9237554a90121c3f47ada Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Wed, 15 Jan 2025 11:52:06 +0100 Subject: [PATCH 02/23] nixos/k3s: use systemd-tmpfiles to activate k3s content Formerly a `ExecStartPre` script was used to link k3s content. Building the script got fairly messy and it had some footguns like forgetting to create parent directories before linking or silent overriding of existing links. --- .../modules/services/cluster/k3s/default.nix | 83 ++++++++++--------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix index 2d182856ab5a..4acc40088e0f 100644 --- a/nixos/modules/services/cluster/k3s/default.nix +++ b/nixos/modules/services/cluster/k3s/default.nix @@ -369,44 +369,6 @@ let }; } ); - - # TODO: use tmpfiles - enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests); - enabledHelmManifests = lib.filter (m: m.enable) (lib.attrValues cfg.autoDeployCharts); - enabledAutoDeployCharts = lib.concatMapAttrs (n: v: { ${n} = v.package; }) ( - lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts - ); - linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}"; - linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}"; - linkChartEntry = - let - mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz"; - in - name: value: - "${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkChartTarget (builtins.baseNameOf name)}"; - - activateK3sContent = pkgs.writeShellScript "activate-k3s-content" '' - ${lib.optionalString ( - builtins.length (enabledManifests ++ enabledHelmManifests) > 0 - ) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"} - ${lib.optionalString ( - cfg.charts != { } || enabledAutoDeployCharts != { } - ) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"} - ${lib.optionalString ( - builtins.length cfg.images > 0 - ) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"} - - ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)} - ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledHelmManifests)} - ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)} - ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry enabledAutoDeployCharts)} - ${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)} - - ${lib.optionalString (cfg.containerdConfigTemplate != null) '' - mkdir -p $(dirname ${containerdConfigTemplateFile}) - ${pkgs.coreutils-full}/bin/ln -sfn ${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate} ${containerdConfigTemplateFile} - ''} - ''; in { imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ]; @@ -826,6 +788,50 @@ in environment.systemPackages = [ config.services.k3s.package ]; + # Use systemd-tmpfiles to activate k3s content + systemd.tmpfiles.settings."10-k3s" = + let + # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests + enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests); + # Merge charts with charts contained in enabled auto deploying charts + helmCharts = + (lib.concatMapAttrs (n: v: { ${n} = v.package; }) ( + lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts + )) + // cfg.charts; + # Make a systemd-tmpfiles rule for a manifest + mkManifestRule = manifest: { + name = "${manifestDir}/${manifest.target}"; + value = { + "L+".argument = "${manifest.source}"; + }; + }; + # Ensure that all chart targets have a .tgz suffix + mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz"; + # Make a systemd-tmpfiles rule for a chart + mkChartRule = target: source: { + name = "${chartDir}/${mkChartTarget target}"; + value = { + "L+".argument = "${source}"; + }; + }; + # Make a systemd-tmpfiles rule for a container image + mkImageRule = image: { + name = "${imageDir}/${image.name}"; + value = { + "L+".argument = "${image}"; + }; + }; + in + (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests) + // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts) + // (builtins.listToAttrs (map mkImageRule cfg.images)) + // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) { + ${containerdConfigTemplateFile} = { + "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}"; + }; + }); + systemd.services.k3s = let kubeletParams = @@ -873,7 +879,6 @@ in LimitCORE = "infinity"; TasksMax = "infinity"; EnvironmentFile = cfg.environmentFile; - ExecStartPre = activateK3sContent; ExecStart = lib.concatStringsSep " \\\n " ( [ "${cfg.package}/bin/k3s ${cfg.role}" ] ++ (lib.optional cfg.clusterInit "--cluster-init") From abfc62b97b33e120094eb69bda4635eb42d24b7c Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Mon, 10 Mar 2025 06:38:15 +0000 Subject: [PATCH 03/23] python312Packages.primer3: 2.0.3 -> 2.1.0 --- pkgs/development/python-modules/primer3/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/primer3/default.nix b/pkgs/development/python-modules/primer3/default.nix index bb4a9cf5d306..b2a18a3e9102 100644 --- a/pkgs/development/python-modules/primer3/default.nix +++ b/pkgs/development/python-modules/primer3/default.nix @@ -13,7 +13,7 @@ buildPythonPackage rec { pname = "primer3"; - version = "2.0.3"; + version = "2.1.0"; pyproject = true; disabled = pythonOlder "3.7"; @@ -22,7 +22,7 @@ buildPythonPackage rec { owner = "libnano"; repo = "primer3-py"; tag = "v${version}"; - hash = "sha256-O8BFjkjG9SfknSrK34s9EJnqTrtCf4zW9A+N+/MHl2w="; + hash = "sha256-Kp4JH57gEdj7SzY+7XGBzGloWuTSwUQRBK9QbgXQfUE="; }; nativeBuildInputs = [ @@ -48,7 +48,7 @@ buildPythonPackage rec { meta = with lib; { description = "Oligo analysis and primer design"; homepage = "https://github.com/libnano/primer3-py"; - changelog = "https://github.com/libnano/primer3-py/blob/v${version}/CHANGES"; + changelog = "https://github.com/libnano/primer3-py/blob/${src.tag}/CHANGES"; license = with licenses; [ gpl2Only ]; maintainers = with maintainers; [ fab ]; }; From cecf361efeb55b061261b050dd35c13c94be2ae6 Mon Sep 17 00:00:00 2001 From: misuzu Date: Mon, 10 Mar 2025 13:48:17 +0200 Subject: [PATCH 04/23] _3proxy: 0.9.4 -> 0.9.5 --- pkgs/by-name/_3/_3proxy/package.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/by-name/_3/_3proxy/package.nix b/pkgs/by-name/_3/_3proxy/package.nix index ad9c639ea45a..7ddbc08a4209 100644 --- a/pkgs/by-name/_3/_3proxy/package.nix +++ b/pkgs/by-name/_3/_3proxy/package.nix @@ -7,13 +7,13 @@ stdenv.mkDerivation rec { pname = "3proxy"; - version = "0.9.4"; + version = "0.9.5"; src = fetchFromGitHub { owner = "3proxy"; repo = pname; rev = version; - sha256 = "sha256-4bLlQ/ULvpjs6fr19yBBln5mRRc+yj+zVLiTs1e/Ypc="; + sha256 = "sha256-uy6flZ1a7o02pr5O0pgl9zCjh8mE9W5JxotJeBMB16A="; }; # They use 'install -s', that calls the native strip instead of the cross. @@ -42,11 +42,11 @@ stdenv.mkDerivation rec { smoke-test = nixosTests._3proxy; }; - meta = with lib; { + meta = { description = "Tiny free proxy server"; homepage = "https://github.com/3proxy/3proxy"; - license = licenses.bsd2; - platforms = platforms.linux; - maintainers = with maintainers; [ misuzu ]; + license = lib.licenses.bsd2; + platforms = lib.platforms.linux; + maintainers = with lib.maintainers; [ misuzu ]; }; } From 0261bc8f135bde924578fc7b46c8c5afd052859a Mon Sep 17 00:00:00 2001 From: Stephen Huan Date: Thu, 13 Mar 2025 14:10:22 -0400 Subject: [PATCH 05/23] python3Packages.wandb: move pydantic to dependencies --- pkgs/development/python-modules/wandb/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/wandb/default.nix b/pkgs/development/python-modules/wandb/default.nix index d724cc4afd6b..32fd5a5b7549 100644 --- a/pkgs/development/python-modules/wandb/default.nix +++ b/pkgs/development/python-modules/wandb/default.nix @@ -25,6 +25,7 @@ platformdirs, protobuf, psutil, + pydantic, pyyaml, requests, sentry-sdk_2, @@ -56,7 +57,6 @@ parameterized, pillow, plotly, - pydantic, pyfakefs, pyte, pytest-asyncio, @@ -184,6 +184,7 @@ buildPythonPackage rec { platformdirs protobuf psutil + pydantic pyyaml requests sentry-sdk_2 @@ -220,7 +221,6 @@ buildPythonPackage rec { parameterized pillow plotly - pydantic pyfakefs pyte pytest-asyncio From b7024f4945dd3f5e3fca633900c5696bc7577bd0 Mon Sep 17 00:00:00 2001 From: Stephen Huan Date: Thu, 13 Mar 2025 14:09:37 -0400 Subject: [PATCH 06/23] python3Packages.wandb: add eval-type-backport to dependencies --- pkgs/development/python-modules/wandb/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/development/python-modules/wandb/default.nix b/pkgs/development/python-modules/wandb/default.nix index 32fd5a5b7549..b9405d748218 100644 --- a/pkgs/development/python-modules/wandb/default.nix +++ b/pkgs/development/python-modules/wandb/default.nix @@ -32,6 +32,7 @@ setproctitle, setuptools, pythonOlder, + eval-type-backport, typing-extensions, # tests @@ -192,6 +193,9 @@ buildPythonPackage rec { # setuptools is necessary since pkg_resources is required at runtime. setuptools ] + ++ lib.optionals (pythonOlder "3.10") [ + eval-type-backport + ] ++ lib.optionals (pythonOlder "3.12") [ typing-extensions ]; From 0f9de7befda169fe8ef42ee75273b9540bde3997 Mon Sep 17 00:00:00 2001 From: Stephen Huan Date: Thu, 13 Mar 2025 14:20:31 -0400 Subject: [PATCH 07/23] python3Packages.wandb: 0.19.6 -> 0.19.8 --- pkgs/development/python-modules/wandb/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/wandb/default.nix b/pkgs/development/python-modules/wandb/default.nix index b9405d748218..91f0c897aa6a 100644 --- a/pkgs/development/python-modules/wandb/default.nix +++ b/pkgs/development/python-modules/wandb/default.nix @@ -76,12 +76,12 @@ }: let - version = "0.19.6"; + version = "0.19.8"; src = fetchFromGitHub { owner = "wandb"; repo = "wandb"; tag = "v${version}"; - hash = "sha256-snyr0IlE4otk1ctWUrJEFAmHYsXe+k6qULCaO3aW0e4="; + hash = "sha256-hveMyGeu9RhdtWMbV/4GQ4KUNfjSt0CKyW7Yx8QtlLM="; }; gpu-stats = rustPlatform.buildRustPackage { From 2820972779aaf69c002457200887a822037daa7e Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 12 Mar 2025 21:24:34 +0100 Subject: [PATCH 08/23] lighthouse: remove unused check input "libpq" --- pkgs/applications/blockchains/lighthouse/default.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkgs/applications/blockchains/lighthouse/default.nix b/pkgs/applications/blockchains/lighthouse/default.nix index 3b91039a5b97..5fac7b05434c 100644 --- a/pkgs/applications/blockchains/lighthouse/default.nix +++ b/pkgs/applications/blockchains/lighthouse/default.nix @@ -7,7 +7,6 @@ , nix-update-script , openssl , pkg-config -, libpq , protobuf , rustPlatform , rust-jemalloc-sys @@ -119,10 +118,6 @@ rustPlatform.buildRustPackage rec { "--skip subnet_service::tests::sync_committee_service::subscribe_and_unsubscribe" ]; - nativeCheckInputs = [ - libpq - ]; - passthru = { tests.version = testers.testVersion { package = lighthouse; From 7dbeb1be3aa9816ab875880791f1da8f2d0d7673 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 12 Mar 2025 21:25:37 +0100 Subject: [PATCH 09/23] various: switch to lighter libpq package instead of postgresql No need for the full server package when you only need libpq. --- .../elisp-packages/elpa-common-overrides.nix | 2 +- pkgs/by-name/bi/biboumi/package.nix | 4 ++-- pkgs/by-name/ke/kea/package.nix | 4 ++-- pkgs/by-name/pg/pg_top/package.nix | 4 ++-- .../development/compilers/chicken/5/overrides.nix | 2 +- pkgs/development/lisp-modules/ql.nix | 2 +- pkgs/development/r-modules/default.nix | 15 +++------------ pkgs/development/tools/tora/default.nix | 2 +- pkgs/kde/gear/akonadi/default.nix | 8 ++++---- 9 files changed, 17 insertions(+), 26 deletions(-) diff --git a/pkgs/applications/editors/emacs/elisp-packages/elpa-common-overrides.nix b/pkgs/applications/editors/emacs/elisp-packages/elpa-common-overrides.nix index cc8c4e1caa21..d3dd6e128681 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/elpa-common-overrides.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/elpa-common-overrides.nix @@ -210,7 +210,7 @@ in poke = addPackageRequires super.poke [ self.poke-mode ]; pq = super.pq.overrideAttrs (old: { - buildInputs = old.buildInputs or [ ] ++ [ pkgs.postgresql ]; + buildInputs = old.buildInputs or [ ] ++ [ pkgs.libpq ]; }); preview-auto = mkHome super.preview-auto; diff --git a/pkgs/by-name/bi/biboumi/package.nix b/pkgs/by-name/bi/biboumi/package.nix index cdcd493aa2f5..cf7f744e2f84 100644 --- a/pkgs/by-name/bi/biboumi/package.nix +++ b/pkgs/by-name/bi/biboumi/package.nix @@ -14,7 +14,7 @@ withIDN ? true, libidn, withPostgreSQL ? false, - postgresql, + libpq, withSQLite ? true, sqlite, withUDNS ? true, @@ -57,7 +57,7 @@ stdenv.mkDerivation rec { botan2 ] ++ lib.optional withIDN libidn - ++ lib.optional withPostgreSQL postgresql + ++ lib.optional withPostgreSQL libpq ++ lib.optional withSQLite sqlite ++ lib.optional withUDNS udns; diff --git a/pkgs/by-name/ke/kea/package.nix b/pkgs/by-name/ke/kea/package.nix index c23db4d772d0..aa8a8a762ecd 100644 --- a/pkgs/by-name/ke/kea/package.nix +++ b/pkgs/by-name/ke/kea/package.nix @@ -15,7 +15,7 @@ libmysqlclient, log4cplus, openssl, - postgresql, + libpq, python3, # tests @@ -54,7 +54,7 @@ stdenv.mkDerivation rec { "--localstatedir=/var" "--with-openssl=${lib.getDev openssl}" ] - ++ lib.optional withPostgres "--with-pgsql=${lib.getDev postgresql}/bin/pg_config" + ++ lib.optional withPostgres "--with-pgsql=${lib.getDev libpq}/bin/pg_config" ++ lib.optional withMysql "--with-mysql=${lib.getDev libmysqlclient}/bin/mysql_config"; postConfigure = '' diff --git a/pkgs/by-name/pg/pg_top/package.nix b/pkgs/by-name/pg/pg_top/package.nix index d004475f736c..4252a38283d5 100644 --- a/pkgs/by-name/pg/pg_top/package.nix +++ b/pkgs/by-name/pg/pg_top/package.nix @@ -4,7 +4,7 @@ lib, libbsd, ncurses, - postgresql, + libpq, stdenv, }: @@ -19,8 +19,8 @@ stdenv.mkDerivation rec { buildInputs = [ libbsd + libpq ncurses - postgresql ]; nativeBuildInputs = [ cmake ]; diff --git a/pkgs/development/compilers/chicken/5/overrides.nix b/pkgs/development/compilers/chicken/5/overrides.nix index 92b1104d9039..9f2570038e1f 100644 --- a/pkgs/development/compilers/chicken/5/overrides.nix +++ b/pkgs/development/compilers/chicken/5/overrides.nix @@ -146,7 +146,7 @@ in ); openssl = addToBuildInputs pkgs.openssl; plot = addToBuildInputs pkgs.plotutils; - postgresql = addToBuildInputsWithPkgConfig pkgs.postgresql; + postgresql = addToBuildInputsWithPkgConfig pkgs.libpq; rocksdb = addToBuildInputs pkgs.rocksdb_8_3; scheme2c-compatibility = addPkgConfig; sdl-base = diff --git a/pkgs/development/lisp-modules/ql.nix b/pkgs/development/lisp-modules/ql.nix index 96baffcf3618..1080eb498a3c 100644 --- a/pkgs/development/lisp-modules/ql.nix +++ b/pkgs/development/lisp-modules/ql.nix @@ -59,7 +59,7 @@ let nativeLibs = [ pkgs.mariadb.client ]; }); clsql-postgresql = super.clsql-postgresql.overrideLispAttrs (o: { - nativeLibs = [ pkgs.postgresql.lib ]; + nativeLibs = [ pkgs.libpq ]; }); clsql-sqlite3 = super.clsql-sqlite3.overrideLispAttrs (o: { nativeLibs = [ pkgs.sqlite ]; diff --git a/pkgs/development/r-modules/default.nix b/pkgs/development/r-modules/default.nix index 2322f0b02d23..34f38d1feba8 100644 --- a/pkgs/development/r-modules/default.nix +++ b/pkgs/development/r-modules/default.nix @@ -335,7 +335,6 @@ let }; packagesWithNativeBuildInputs = { - adbcpostgresql = [ pkgs.postgresql ]; adimpro = [ pkgs.imagemagick ]; animation = [ pkgs.which ]; Apollonius = with pkgs; [ pkg-config gmp.dev mpfr.dev ]; @@ -484,8 +483,7 @@ let RODBC = [ pkgs.libiodbc ]; rpanel = [ pkgs.tclPackages.bwidget ]; Rpoppler = [ pkgs.poppler ]; - RPostgres = with pkgs; [ postgresql ]; - RPostgreSQL = with pkgs; [ postgresql postgresql ]; + RPostgreSQL = with pkgs; [ libpq ]; RProtoBuf = [ pkgs.protobuf ]; RSclient = [ pkgs.openssl.dev ]; Rserve = [ pkgs.openssl ]; @@ -614,7 +612,7 @@ let packagesWithBuildInputs = { # sort -t '=' -k 2 - adbcpostgresql = with pkgs; [ readline.dev zlib.dev openssl.dev libkrb5.dev openpam ]; + adbcpostgresql = with pkgs; [ readline.dev zlib.dev openssl.dev libkrb5.dev openpam libpq ]; asciicast = with pkgs; [ xz.dev bzip2.dev zlib.dev icu.dev libdeflate ]; island = [ pkgs.gsl.dev ]; svKomodo = [ pkgs.which ]; @@ -646,6 +644,7 @@ let RGtk2 = [ pkgs.pkg-config ]; RProtoBuf = [ pkgs.pkg-config ]; Rpoppler = [ pkgs.pkg-config ]; + RPostgres = with pkgs; [ libpq ]; XML = [ pkgs.pkg-config ]; apsimx = [ pkgs.which ]; cairoDevice = [ pkgs.pkg-config ]; @@ -1625,14 +1624,6 @@ let enableParallelBuilding = false; }); - RPostgres = old.RPostgres.overrideAttrs (attrs: { - preConfigure = '' - export INCLUDE_DIR=${pkgs.postgresql}/include - export LIB_DIR=${pkgs.postgresql.lib}/lib - patchShebangs configure - ''; - }); - OpenMx = old.OpenMx.overrideAttrs (attrs: { env = (attrs.env or { }) // { # needed to avoid "log limit exceeded" on Hydra diff --git a/pkgs/development/tools/tora/default.nix b/pkgs/development/tools/tora/default.nix index fde6ca673de1..88482eb91e3c 100644 --- a/pkgs/development/tools/tora/default.nix +++ b/pkgs/development/tools/tora/default.nix @@ -42,7 +42,7 @@ mkDerivation { loki libmysqlclient openssl - postgresql + postgresql # needs libecpg, which is not available in libpq package qscintilla qtbase ]; diff --git a/pkgs/kde/gear/akonadi/default.nix b/pkgs/kde/gear/akonadi/default.nix index e54e8ae37e4c..6225b7e9cfda 100644 --- a/pkgs/kde/gear/akonadi/default.nix +++ b/pkgs/kde/gear/akonadi/default.nix @@ -7,7 +7,7 @@ shared-mime-info, xz, mariadb, - postgresql, + libpq, sqlite, backend ? "mysql", }: @@ -32,7 +32,7 @@ mkKdeDerivation { "-DMYSQLD_SCRIPTS_PATH=${lib.getBin mariadb}/bin" ] ++ lib.optionals (backend == "postgres") [ - "-DPOSTGRES_PATH=${lib.getBin postgresql}/bin" + "-DPOSTGRES_PATH=${lib.getBin libpq}/bin" ]; extraNativeBuildInputs = [ @@ -47,7 +47,7 @@ mkKdeDerivation { xz ] ++ lib.optionals (backend == "mysql") [ mariadb ] - ++ lib.optionals (backend == "postgres") [ postgresql ] + ++ lib.optionals (backend == "postgres") [ libpq ] ++ lib.optionals (backend == "sqlite") [ sqlite ]; # Hardcoded as a QString, which is UTF-16 so Nix can't pick it up automatically @@ -60,6 +60,6 @@ mkKdeDerivation { echo "${mariadb}" > $out/nix-support/depends '' + lib.optionalString (backend == "postgres") '' - echo "${postgresql}" > $out/nix-support/depends + echo "${libpq}" > $out/nix-support/depends ''; } From 96f4a027ad9791bbde14e3405568c4d103584808 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 12 Mar 2025 21:26:04 +0100 Subject: [PATCH 10/23] python3Packages.pgsanity: fix build This needs postgresql instead of libpq, because it depends on the ecpg binary. --- pkgs/development/python-modules/pgsanity/default.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pgsanity/default.nix b/pkgs/development/python-modules/pgsanity/default.nix index 19c22876b93d..c9bb47e33d1e 100644 --- a/pkgs/development/python-modules/pgsanity/default.nix +++ b/pkgs/development/python-modules/pgsanity/default.nix @@ -2,7 +2,7 @@ lib, fetchPypi, buildPythonPackage, - libpq, + postgresql, unittestCheckHook, }: @@ -22,7 +22,10 @@ buildPythonPackage rec { unittestFlagsArray = [ "test" ]; - propagatedBuildInputs = [ libpq ]; + propagatedBuildInputs = [ postgresql ]; + + # To find "ecpg" + nativeBuildInputs = [ (lib.getDev postgresql) ]; meta = with lib; { homepage = "https://github.com/markdrago/pgsanity"; From bd3c8343e5014c991ea7bfe46a9d77dff1b7b608 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 12 Mar 2025 21:26:29 +0100 Subject: [PATCH 11/23] glom: bump to latest postgresql dependency --- pkgs/by-name/gl/glom/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/gl/glom/package.nix b/pkgs/by-name/gl/glom/package.nix index a7e532ea116b..3da677d706ba 100644 --- a/pkgs/by-name/gl/glom/package.nix +++ b/pkgs/by-name/gl/glom/package.nix @@ -31,7 +31,7 @@ isocodes, gtksourceview, gtksourceviewmm, - postgresql_15, + postgresql, gobject-introspection, yelp-tools, wrapGAppsHook3, @@ -119,7 +119,7 @@ stdenv.mkDerivation (finalAttrs: { isocodes gtksourceview gtksourceviewmm - postgresql_15 # for postgresql utils + postgresql # for postgresql utils ]; enableParallelBuilding = true; @@ -128,7 +128,7 @@ stdenv.mkDerivation (finalAttrs: { configureFlags = [ "--with-boost-python=boost_python${lib.versions.major python311.version}${lib.versions.minor python311.version}" - "--with-postgres-utils=${lib.getBin postgresql_15}/bin" + "--with-postgres-utils=${lib.getBin postgresql}/bin" ]; makeFlags = [ From e64f47c5b2c906239a94e37da84fbc9209f04f94 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 12 Mar 2025 21:26:56 +0100 Subject: [PATCH 12/23] pkg-config-data: provide libpq via libpq by default, not postgresql --- pkgs/top-level/pkg-config/pkg-config-data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/pkg-config/pkg-config-data.json b/pkgs/top-level/pkg-config/pkg-config-data.json index 60981baa69cf..5705c8ad4c70 100644 --- a/pkgs/top-level/pkg-config/pkg-config-data.json +++ b/pkgs/top-level/pkg-config/pkg-config-data.json @@ -525,7 +525,7 @@ }, "libpq": { "attrPath": [ - "postgresql" + "libpq" ] }, "libpulse": { From a7a3283aa3c60c2a37975cd38da37af22093b022 Mon Sep 17 00:00:00 2001 From: Albert Ilagan Date: Sat, 15 Mar 2025 03:22:37 +0800 Subject: [PATCH 13/23] heimdall-proxy: 0.15.8 -> 0.15.9 --- pkgs/by-name/he/heimdall-proxy/package.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/by-name/he/heimdall-proxy/package.nix b/pkgs/by-name/he/heimdall-proxy/package.nix index 9a8260e6ea94..03795945f2a1 100644 --- a/pkgs/by-name/he/heimdall-proxy/package.nix +++ b/pkgs/by-name/he/heimdall-proxy/package.nix @@ -1,12 +1,12 @@ { fetchFromGitHub, - buildGo124Module, + buildGoModule, lib, }: let - version = "0.15.8"; + version = "0.15.9"; in -buildGo124Module { +buildGoModule { pname = "heimdall-proxy"; inherit version; @@ -15,10 +15,10 @@ buildGo124Module { owner = "dadrus"; repo = "heimdall"; tag = "v${version}"; - hash = "sha256-UUQWYChZEb/5mc2YYwIJSQ+pCUXIwvB09KaR0FoKrA4="; + hash = "sha256-nrYeNVSDvGTRywhTLFLylnSz1jhR/1OSKDaRj2sDe5o="; }; - vendorHash = "sha256-4bnVqUV3H/mZ9FiApZk6pVbRWAqpy17+/dGxXR0fjW0="; + vendorHash = "sha256-Rz1v2jusP9edDpoFaiwb7ZatuSeg9sqFS7j2JZtNJio="; tags = [ "sqlite" ]; From a295e27aebe95785911504369ffe0a6c83bf76ea Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Fri, 14 Mar 2025 21:03:06 +0000 Subject: [PATCH 14/23] python312Packages.weblate-language-data: 2025.2 -> 2025.3 --- .../python-modules/weblate-language-data/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/weblate-language-data/default.nix b/pkgs/development/python-modules/weblate-language-data/default.nix index b1301c6237be..b41d626e8952 100644 --- a/pkgs/development/python-modules/weblate-language-data/default.nix +++ b/pkgs/development/python-modules/weblate-language-data/default.nix @@ -8,13 +8,13 @@ buildPythonPackage rec { pname = "weblate-language-data"; - version = "2025.2"; + version = "2025.3"; pyproject = true; src = fetchPypi { pname = "weblate_language_data"; inherit version; - hash = "sha256-T3O107CQ01loE68vlQtcCjeytxCSiu0m5Oj5P06z2NU="; + hash = "sha256-1uZqqwJds+Q2yL2OP2dEEbp4sJmJN28gOYDDJ3fhBRA="; }; build-system = [ setuptools ]; From ec734bdd81835f8eed5f66c09819a8d155d12b08 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 15 Mar 2025 00:06:04 +0000 Subject: [PATCH 15/23] python312Packages.unstructured-inference: 0.8.7 -> 0.8.9 --- .../python-modules/unstructured-inference/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/unstructured-inference/default.nix b/pkgs/development/python-modules/unstructured-inference/default.nix index d564aa2c6bec..8c515ef98abf 100644 --- a/pkgs/development/python-modules/unstructured-inference/default.nix +++ b/pkgs/development/python-modules/unstructured-inference/default.nix @@ -23,14 +23,14 @@ buildPythonPackage rec { pname = "unstructured-inference"; - version = "0.8.7"; + version = "0.8.9"; format = "setuptools"; src = fetchFromGitHub { owner = "Unstructured-IO"; repo = "unstructured-inference"; tag = version; - hash = "sha256-uH7LDezHZrD1zeWMDzrZZALGf0oocIVZl68MactBFGQ="; + hash = "sha256-4wfZFu0551jbpeSYq6RHrDpThm+B2tygVwLlggPkbog="; }; propagatedBuildInputs = From b621de872f44fbad4a2381dea189e4dbc55df5ad Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 15 Mar 2025 02:02:45 +0000 Subject: [PATCH 16/23] redpanda-client: 24.3.6 -> 24.3.7 --- pkgs/by-name/re/redpanda-client/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/re/redpanda-client/package.nix b/pkgs/by-name/re/redpanda-client/package.nix index 734b4b93d1a3..10a3ca189e77 100644 --- a/pkgs/by-name/re/redpanda-client/package.nix +++ b/pkgs/by-name/re/redpanda-client/package.nix @@ -7,12 +7,12 @@ stdenv, }: let - version = "24.3.6"; + version = "24.3.7"; src = fetchFromGitHub { owner = "redpanda-data"; repo = "redpanda"; rev = "v${version}"; - sha256 = "sha256-OxaAWQKa1rmMf0F/Pu0ZH0DYG5UIEri/RBYJ4fEgkYI="; + sha256 = "sha256-2FpJMJau5rrKLUaW7nrE7pMf0zi2cW64+Hlk2lPvHxY="; }; in buildGoModule rec { @@ -20,7 +20,7 @@ buildGoModule rec { inherit doCheck src version; modRoot = "./src/go/rpk"; runVend = false; - vendorHash = "sha256-VR8HvU8aPBWaKH+xjn+2CO794cipB7mOJCkXtGYGsdk="; + vendorHash = "sha256-MdfCc3XdoMv3nnyaCbqU7mwJSgtusw9wVWjYqqJJmHA="; ldflags = [ ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/version.version=${version}"'' From 6fd0455eb3328b6ed5d4c1cb75128a9bc302d727 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 15 Mar 2025 02:43:39 +0000 Subject: [PATCH 17/23] python312Packages.elasticsearch8: 8.17.1 -> 8.17.2 --- pkgs/development/python-modules/elasticsearch8/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/elasticsearch8/default.nix b/pkgs/development/python-modules/elasticsearch8/default.nix index 7319090d0cd0..2e4deb6c103a 100644 --- a/pkgs/development/python-modules/elasticsearch8/default.nix +++ b/pkgs/development/python-modules/elasticsearch8/default.nix @@ -12,14 +12,14 @@ buildPythonPackage rec { pname = "elasticsearch8"; - version = "8.17.1"; + version = "8.17.2"; pyproject = true; disabled = pythonOlder "3.7"; src = fetchPypi { inherit pname version; - hash = "sha256-u0FLfj2Dh2l5RiCPDwtuXnm88WXNAhI0sux+UgDTNpA="; + hash = "sha256-j6FaQWPFJ8kqoTwjIPyMDcOZBg8mOO0BbKCFn4ESCAM="; }; build-system = [ hatchling ]; From f7a112205027ea066c90b7764ea888768a3794cf Mon Sep 17 00:00:00 2001 From: Curtis Jones Date: Sat, 15 Mar 2025 00:30:12 -0400 Subject: [PATCH 18/23] paru: correct shell completion file names Previously the package was installing blank shell completion files. i.e: `share/zsh/site-functions/_zsh` instead of `share/zsh/site-functions/_paru` --- pkgs/by-name/pa/paru/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/pa/paru/package.nix b/pkgs/by-name/pa/paru/package.nix index 8f8780a8a548..52278491e4ed 100644 --- a/pkgs/by-name/pa/paru/package.nix +++ b/pkgs/by-name/pa/paru/package.nix @@ -53,9 +53,9 @@ rustPlatform.buildRustPackage rec { postInstall = '' installManPage man/paru.8 man/paru.conf.5 - installShellCompletion --bash completions/bash - installShellCompletion --fish completions/fish - installShellCompletion --zsh completions/zsh + installShellCompletion --name paru.bash --bash completions/bash + installShellCompletion --name paru.fish --fish completions/fish + installShellCompletion --name _paru --zsh completions/zsh cp -r locale "$out/share/" ''; From 707424f8c16cb58ad0c1d089b3f2bc9448e681bd Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 15 Mar 2025 07:00:38 +0000 Subject: [PATCH 19/23] flet-client-flutter: 0.27.4 -> 0.27.6 --- pkgs/by-name/fl/flet-client-flutter/package.nix | 4 ++-- pkgs/by-name/fl/flet-client-flutter/pubspec.lock.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/fl/flet-client-flutter/package.nix b/pkgs/by-name/fl/flet-client-flutter/package.nix index 65476713f703..11f86847048b 100644 --- a/pkgs/by-name/fl/flet-client-flutter/package.nix +++ b/pkgs/by-name/fl/flet-client-flutter/package.nix @@ -19,13 +19,13 @@ flutter327.buildFlutterApplication rec { pname = "flet-client-flutter"; - version = "0.27.4"; + version = "0.27.6"; src = fetchFromGitHub { owner = "flet-dev"; repo = "flet"; tag = "v${version}"; - hash = "sha256-YwRxkubkpFHKmTzb+RKZFumey3153lAtg2NO3dDKWG4="; + hash = "sha256-ZtIAfLdj9209ZzgmNzTHMyzCTohxYK0Va4M8NYyie64="; }; sourceRoot = "${src.name}/client"; diff --git a/pkgs/by-name/fl/flet-client-flutter/pubspec.lock.json b/pkgs/by-name/fl/flet-client-flutter/pubspec.lock.json index 311272748712..ef2b7145add1 100644 --- a/pkgs/by-name/fl/flet-client-flutter/pubspec.lock.json +++ b/pkgs/by-name/fl/flet-client-flutter/pubspec.lock.json @@ -327,7 +327,7 @@ "relative": true }, "source": "path", - "version": "0.27.1" + "version": "0.27.5" }, "flet_ads": { "dependency": "direct main", From 464ba64341494532ea703f50c0b1f1e3246834d7 Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Sat, 15 Mar 2025 08:10:09 +0100 Subject: [PATCH 20/23] python312Packages.weblate-language-data: add changelog to meta --- .../development/python-modules/weblate-language-data/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/weblate-language-data/default.nix b/pkgs/development/python-modules/weblate-language-data/default.nix index b41d626e8952..bf8f780f29cc 100644 --- a/pkgs/development/python-modules/weblate-language-data/default.nix +++ b/pkgs/development/python-modules/weblate-language-data/default.nix @@ -29,6 +29,7 @@ buildPythonPackage rec { meta = with lib; { description = "Language definitions used by Weblate"; homepage = "https://github.com/WeblateOrg/language-data"; + changelog = "https://github.com/WeblateOrg/language-data/releases/tag/${version}"; license = licenses.mit; maintainers = with maintainers; [ erictapen ]; }; From 45c19a484ad6d436fefd0e99a04bc2aaddd06d2c Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Sat, 15 Mar 2025 08:15:19 +0100 Subject: [PATCH 21/23] python312Packages.primer3: refactor --- pkgs/development/python-modules/primer3/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/development/python-modules/primer3/default.nix b/pkgs/development/python-modules/primer3/default.nix index b2a18a3e9102..12fe9407cb73 100644 --- a/pkgs/development/python-modules/primer3/default.nix +++ b/pkgs/development/python-modules/primer3/default.nix @@ -25,10 +25,9 @@ buildPythonPackage rec { hash = "sha256-Kp4JH57gEdj7SzY+7XGBzGloWuTSwUQRBK9QbgXQfUE="; }; - nativeBuildInputs = [ - cython - setuptools - ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ gcc ]; + build-system = [ setuptools ]; + + nativeBuildInputs = [ cython ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ gcc ]; nativeCheckInputs = [ click @@ -49,7 +48,7 @@ buildPythonPackage rec { description = "Oligo analysis and primer design"; homepage = "https://github.com/libnano/primer3-py"; changelog = "https://github.com/libnano/primer3-py/blob/${src.tag}/CHANGES"; - license = with licenses; [ gpl2Only ]; + license = licenses.gpl2Only; maintainers = with maintainers; [ fab ]; }; } From 2371f38c394f10a830670ff230ed9de757413d6e Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 15 Mar 2025 08:51:31 +0000 Subject: [PATCH 22/23] typos: 1.30.0 -> 1.30.2 --- pkgs/by-name/ty/typos/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ty/typos/package.nix b/pkgs/by-name/ty/typos/package.nix index 3ef7ed582a1e..5b4f8fa11d8c 100644 --- a/pkgs/by-name/ty/typos/package.nix +++ b/pkgs/by-name/ty/typos/package.nix @@ -8,17 +8,17 @@ rustPlatform.buildRustPackage rec { pname = "typos"; - version = "1.30.0"; + version = "1.30.2"; src = fetchFromGitHub { owner = "crate-ci"; repo = "typos"; tag = "v${version}"; - hash = "sha256-SFE6hieK2SU+Dmf0eDc35/INJKPoySUJBE9ES8KrCIg="; + hash = "sha256-Dayr+mskYmbLY0yE0OLreMjy8rbyoqY3rgREtaO3+D8="; }; useFetchCargoVendor = true; - cargoHash = "sha256-Cnk/iLdkDyxoHU+vRxuLIyGkcqaGF8WjVOnhNoxg3T4="; + cargoHash = "sha256-K5ekHIfQQxjkydghoU/8pBnzt/q8hSrYFYf1c4GInBM="; passthru.updateScript = nix-update-script { }; From 38590f303b54b358a2cfe788b238fff588e389f7 Mon Sep 17 00:00:00 2001 From: jrdsgl Date: Sat, 15 Mar 2025 02:59:05 -0700 Subject: [PATCH 23/23] nixos/changedetection-io: fix typo (#383539) Update changedetection-io.nix typo correction variables --- nixos/modules/services/web-apps/changedetection-io.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/web-apps/changedetection-io.nix b/nixos/modules/services/web-apps/changedetection-io.nix index 5fd340a0ead0..ff710d55d40c 100644 --- a/nixos/modules/services/web-apps/changedetection-io.nix +++ b/nixos/modules/services/web-apps/changedetection-io.nix @@ -73,7 +73,7 @@ in default = null; example = "/run/secrets/changedetection-io.env"; description = '' - Securely pass environment variabels to changedetection-io. + Securely pass environment variables to changedetection-io. This can be used to set for example a frontend password reproducible via `SALTED_PASS` which convinetly also deactivates nags about the hosted version.