From bbcccf995a829486fc3f35a9f0cb701329600186 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 6 Jun 2024 00:14:53 +0100 Subject: [PATCH 1/9] =?UTF-8?q?kminion:=202.2.7=20=E2=86=92=202.2.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/km/kminion/package.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 2575ff228e1b..1f642cc5670d 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -6,16 +6,16 @@ buildGoModule { pname = "kafka-minion"; - version = "2.2.7"; + version = "2.2.8"; src = fetchFromGitHub { owner = "redpanda-data"; repo = "kminion"; - rev = "0c90d4301ed4600d1aaf3345b6f16587d2f282fc"; - hash = "sha256-CWjX46Sfc9Xj+R7+CZeMuTY0iUStzyZXI4FotwqR44M="; + rev = "v${version}"; + hash = "sha256-y9FcvWJ9izS5vkgDsiHa8TKdS4jOYLMOHJUyCPsmTZ4="; }; - vendorHash = "sha256-6yfQVoY/bHMA4s0IN5ltnQdHWnE3kIKza36uEcGa11U="; + vendorHash = "sha256-ekOS16B2AIB4LQXOLbiaOMUvtqy8f51UJfu0uhn4gzg="; doCheck = false; From d2163bc1aac323d2f28bbed5e494e9d9effca607 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 6 Jun 2024 00:37:11 +0100 Subject: [PATCH 2/9] kminion: Added ldflags for version --- pkgs/by-name/km/kminion/package.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 1f642cc5670d..0431ae49ed02 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -19,6 +19,13 @@ buildGoModule { doCheck = false; + ldflags = [ + "-s" + "-w" + "-X main.version=${version}" + "-X main.builtAt=unknown" + ]; + passthru.updateScript = ./update.sh; meta = with lib; { From 314b467ac176495e094232a3596a4c0396cb1df8 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 6 Jun 2024 00:40:06 +0100 Subject: [PATCH 3/9] nixos/prometheus-exporters: Added kafka exporter --- .../monitoring/prometheus/exporters.nix | 1 + .../monitoring/prometheus/exporters/kafka.nix | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 nixos/modules/services/monitoring/prometheus/exporters/kafka.nix diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix index 28144b6267c2..35a89ab1bf71 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters.nix @@ -79,6 +79,7 @@ let "jitsi" "json" "junos-czerwonk" + "kafka" "kea" "keylight" "klipper" diff --git a/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix b/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix new file mode 100644 index 000000000000..6fa206d3dae2 --- /dev/null +++ b/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix @@ -0,0 +1,45 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.prometheus.exporters.kafka; + inherit (lib) + mkIf + mkOption + mkMerge + types + concatStringsSep + ; +in { + port = 8080; + + extraOpts = { + package = lib.mkPackageOption pkgs "kminion" { }; + + environmentFile = mkOption { + type = with types; nullOr path; + default = null; + description = '' + File containing the credentials to access the repository, in the + format of an EnvironmentFile as described by systemd.exec(5) + ''; + }; + }; + serviceOpts = mkMerge ([{ + serviceConfig = { + ExecStart = '' + ${lib.getExe cfg.package} + ''; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + RestartSec = "5s"; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + }; + }] ++ [ + (mkIf config.services.apache-kafka.enable { + after = [ "apache-kafka.service" ]; + requires = [ "apache-kafka.service" ]; + }) + ]); +} From 8ff6516c3c92c06e91cdac9e80eba75e94ae8c98 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 6 Jun 2024 00:40:30 +0100 Subject: [PATCH 4/9] nixos/prometheus-exporters: Added kafka test --- nixos/tests/prometheus-exporters.nix | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index 010e912b563a..cbfd5426af07 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -587,6 +587,63 @@ let ''; }; + kafka = { + exporterConfig = { + enable = true; + environmentFile = pkgs.writeTextFile { + name = "/tmp/prometheus-kafka-exporter.env"; + text = '' + KAFKA_BROKERS="localhost:9092" + ''; + }; + }; + metricProvider = { + services.apache-kafka = { + enable = true; + + clusterId = "pHG8aWuXSfWAibHFDCnsCQ"; + + formatLogDirs = true; + + settings = { + "node.id" = 1; + "process.roles" = [ + "broker" + "controller" + ]; + "listeners" = [ + "PLAINTEXT://:9092" + "CONTROLLER://:9093" + ]; + "listener.security.protocol.map" = [ + "PLAINTEXT:PLAINTEXT" + "CONTROLLER:PLAINTEXT" + ]; + "controller.quorum.voters" = [ + "1@localhost:9093" + ]; + "controller.listener.names" = [ "CONTROLLER" ]; + "log.dirs" = [ "/var/lib/apache-kafka" ]; + }; + }; + + systemd.services.apache-kafka.serviceConfig.StateDirectory = "apache-kafka"; + }; + exporterTest = '' + wait_for_unit("apache-kafka") + wait_for_open_port(9092) + wait_for_open_port(9093) + wait_for_unit("prometheus-kafka-exporter.service") + wait_for_open_port(8080) + wait_until_succeeds( + "journalctl -o cat -u prometheus-kafka-exporter.service | grep '\"version\":\"${pkgs.kminion.version}\"'" + ) + succeed( + "curl -sSf http://localhost:8080/metrics | grep 'kminion_exporter_up'" + ) + ''; + }; + knot = { exporterConfig = { enable = true; From eba6963678703d26db3242c4c9982f8191b59fd7 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 10 Jun 2024 14:13:48 +0100 Subject: [PATCH 5/9] doc/release-notes: 25.11: Added KMinion item to new services. --- nixos/doc/manual/release-notes/rl-2511.section.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2511.section.md b/nixos/doc/manual/release-notes/rl-2511.section.md index 15fd7c573239..db028db407e9 100644 --- a/nixos/doc/manual/release-notes/rl-2511.section.md +++ b/nixos/doc/manual/release-notes/rl-2511.section.md @@ -92,6 +92,8 @@ - [conman](https://github.com/dun/conman), a serial console management program. Available as [services.conman](#opt-services.conman.enable). +- [KMinion](https://github.com/redpanda-data/kminion), feature-rich Prometheus exporter for Apache Kafka. Available as [services.prometheus.exporters.kafka](options.html#opt-services.prometheus.exporters.kafka). + ## Backward Incompatibilities {#sec-release-25.11-incompatibilities} From f39852844f8a5d3a72a1479ea88b097950348326 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 10 Jun 2024 14:26:40 +0100 Subject: [PATCH 6/9] kminion: Added nixosTests to passthru.tests --- pkgs/by-name/km/kminion/package.nix | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 0431ae49ed02..6e87c0cb5cdf 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -1,7 +1,8 @@ -{ - lib, - fetchFromGitHub, - buildGoModule, +{ lib +, stdenv +, fetchFromGitHub +, buildGoModule +, nixosTests }: buildGoModule { @@ -26,7 +27,12 @@ buildGoModule { "-X main.builtAt=unknown" ]; - passthru.updateScript = ./update.sh; + passthru = { + tests = { + inherit (nixosTests.prometheus-exporters) kafka; + }; + updateScript = ./update.sh; + }; meta = with lib; { description = "Feature-rich Prometheus exporter for Apache Kafka written in Go"; From 1dfbaec1bafc63c26b1cb0706c1ccc2a64cc409c Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 13 Jun 2024 08:57:44 +0100 Subject: [PATCH 7/9] kminion: Removed platform restriction --- pkgs/by-name/km/kminion/package.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 6e87c0cb5cdf..61b51836e4c5 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -37,7 +37,6 @@ buildGoModule { meta = with lib; { description = "Feature-rich Prometheus exporter for Apache Kafka written in Go"; license = licenses.mit; - platforms = platforms.linux; maintainers = with maintainers; [ cafkafk ]; mainProgram = "kminion"; }; From 8373b658567fd15bde88bb100fc9bedaeecc51d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Sun, 29 Dec 2024 13:55:08 +0100 Subject: [PATCH 8/9] kminion: format source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christina Sørensen --- .../monitoring/prometheus/exporters/kafka.nix | 51 +++++++++++-------- pkgs/by-name/km/kminion/package.nix | 11 ++-- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix b/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix index 6fa206d3dae2..ebe17e08aeac 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters/kafka.nix @@ -1,4 +1,9 @@ -{ config, lib, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: let cfg = config.services.prometheus.exporters.kafka; @@ -9,7 +14,8 @@ let types concatStringsSep ; -in { +in +{ port = 8080; extraOpts = { @@ -24,22 +30,27 @@ in { ''; }; }; - serviceOpts = mkMerge ([{ - serviceConfig = { - ExecStart = '' - ${lib.getExe cfg.package} - ''; - EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; - RestartSec = "5s"; - RestrictAddressFamilies = [ - "AF_INET" - "AF_INET6" - ]; - }; - }] ++ [ - (mkIf config.services.apache-kafka.enable { - after = [ "apache-kafka.service" ]; - requires = [ "apache-kafka.service" ]; - }) - ]); + serviceOpts = mkMerge ( + [ + { + serviceConfig = { + ExecStart = '' + ${lib.getExe cfg.package} + ''; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + RestartSec = "5s"; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + }; + } + ] + ++ [ + (mkIf config.services.apache-kafka.enable { + after = [ "apache-kafka.service" ]; + requires = [ "apache-kafka.service" ]; + }) + ] + ); } diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 61b51836e4c5..053db4f4bfa3 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -1,8 +1,9 @@ -{ lib -, stdenv -, fetchFromGitHub -, buildGoModule -, nixosTests +{ + lib, + stdenv, + fetchFromGitHub, + buildGoModule, + nixosTests, }: buildGoModule { From f482bcebdf8bc00d93b7f7a0fd7c2c6c165069f5 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 8 Jul 2025 11:06:24 +0000 Subject: [PATCH 9/9] =?UTF-8?q?kminion:=202.2.8=20=E2=86=92=202.2.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/km/kminion/package.nix | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkgs/by-name/km/kminion/package.nix b/pkgs/by-name/km/kminion/package.nix index 053db4f4bfa3..9d9f8b150624 100644 --- a/pkgs/by-name/km/kminion/package.nix +++ b/pkgs/by-name/km/kminion/package.nix @@ -1,30 +1,29 @@ { lib, - stdenv, fetchFromGitHub, buildGoModule, nixosTests, }: -buildGoModule { +buildGoModule (finalAttrs: { pname = "kafka-minion"; - version = "2.2.8"; + version = "2.2.13"; src = fetchFromGitHub { owner = "redpanda-data"; repo = "kminion"; - rev = "v${version}"; - hash = "sha256-y9FcvWJ9izS5vkgDsiHa8TKdS4jOYLMOHJUyCPsmTZ4="; + rev = "v${finalAttrs.version}"; + hash = "sha256-s7kHMMU/srqww/N5szTvX6hOFDV9k9hm+0EZUxIj9So="; }; - vendorHash = "sha256-ekOS16B2AIB4LQXOLbiaOMUvtqy8f51UJfu0uhn4gzg="; + vendorHash = "sha256-vdbSKEWlFH4UkuBxu0LFs8+Rwa4aWTjE8gD4zKuvcs4="; doCheck = false; ldflags = [ "-s" "-w" - "-X main.version=${version}" + "-X main.version=${finalAttrs.version}" "-X main.builtAt=unknown" ]; @@ -41,4 +40,4 @@ buildGoModule { maintainers = with maintainers; [ cafkafk ]; mainProgram = "kminion"; }; -} +})