nixos/prometheus/alertmanager-ntfy: init module (#402291)

This commit is contained in:
Sandro
2025-05-05 13:38:24 +02:00
committed by GitHub
13 changed files with 944 additions and 676 deletions
@@ -166,6 +166,8 @@
- [OliveTin](https://www.olivetin.app/), gives safe and simple access to predefined shell commands from a web interface. Available as [services.olivetin](#opt-services.olivetin.enable).
- [alertmanager-ntfy](https://github.com/alexbakker/alertmanager-ntfy), forwards Prometheus Alertmanager notifications to ntfy.sh. Available as [services.prometheus.alertmanager-ntfy](#opt-services.prometheus.alertmanager-ntfy.enable).
- [Stash](https://github.com/stashapp/stash), An organizer for your adult videos/images, written in Go. Available as [services.stash](#opt-services.stash.enable).
- [vsmartcard-vpcd](https://frankmorgner.github.io/vsmartcard/virtualsmartcard/README.html), a virtual smart card driver. Available as [services.vsmartcard-vpcd](#opt-services.vsmartcard-vpcd.enable).
+1
View File
@@ -985,6 +985,7 @@
./services/monitoring/pgscv.nix
./services/monitoring/prometheus/alertmanager-gotify-bridge.nix
./services/monitoring/prometheus/alertmanager-irc-relay.nix
./services/monitoring/prometheus/alertmanager-ntfy.nix
./services/monitoring/prometheus/alertmanager-webhook-logger.nix
./services/monitoring/prometheus/alertmanager.nix
./services/monitoring/prometheus/default.nix
@@ -0,0 +1,201 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.prometheus.alertmanager-ntfy;
settingsFormat = pkgs.formats.yaml { };
settingsFile = settingsFormat.generate "settings.yml" cfg.settings;
configsArg = lib.concatStringsSep "," (
[ settingsFile ] ++ lib.imap0 (i: _: "%d/config-${toString i}.yml") cfg.extraConfigFiles
);
in
{
meta.maintainers = with lib.maintainers; [ defelo ];
options.services.prometheus.alertmanager-ntfy = {
enable = lib.mkEnableOption "alertmanager-ntfy";
package = lib.mkPackageOption pkgs "alertmanager-ntfy" { };
settings = lib.mkOption {
description = ''
Configuration of alertmanager-ntfy.
See <https://github.com/alexbakker/alertmanager-ntfy> for more information.
'';
default = { };
type = lib.types.submodule {
freeformType = settingsFormat.type;
options = {
http.addr = lib.mkOption {
type = lib.types.str;
description = "The address to listen on.";
default = "127.0.0.1:8000";
example = ":8000";
};
ntfy = {
baseurl = lib.mkOption {
type = lib.types.str;
description = "The base URL of the ntfy.sh instance.";
example = "https://ntfy.sh";
};
notification = {
topic = lib.mkOption {
type = lib.types.str;
description = ''
The topic to which alerts should be published.
Can either be a hardcoded string or a gval expression that evaluates to a string.
'';
example = "alertmanager";
};
priority = lib.mkOption {
type = lib.types.str;
description = ''
The ntfy.sh message priority (see <https://docs.ntfy.sh/publish/#message-priority> for more information).
Can either be a hardcoded string or a gval expression that evaluates to a string.
'';
default = ''status == "firing" ? "high" : "default"'';
};
tags = lib.mkOption {
type = lib.types.listOf (
lib.types.submodule {
options = {
tag = lib.mkOption {
type = lib.types.str;
description = ''
The tag to add.
See <https://docs.ntfy.sh/emojis> for a list of all supported emojis.
'';
example = "rotating_light";
};
condition = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
The condition under which this tag should be added.
Tags with no condition are always included.
'';
default = null;
example = ''status == "firing"'';
};
};
}
);
description = ''
Tags to add to ntfy.sh messages.
See <https://docs.ntfy.sh/publish/#tags-emojis> for more information.
'';
default = [
{
tag = "green_circle";
condition = ''status == "resolved"'';
}
{
tag = "red_circle";
condition = ''status == "firing"'';
}
];
};
templates = {
title = lib.mkOption {
type = lib.types.str;
description = "The ntfy.sh message title template.";
default = ''
{{ if eq .Status "resolved" }}Resolved: {{ end }}{{ index .Annotations "summary" }}
'';
};
description = lib.mkOption {
type = lib.types.str;
description = "The ntfy.sh message description template.";
default = ''
{{ index .Annotations "description" }}
'';
};
};
};
};
};
};
};
extraConfigFiles = lib.mkOption {
type = lib.types.listOf lib.types.path;
default = [ ];
example = [ "/run/secrets/alertmanager-ntfy.yml" ];
description = ''
Config files to merge into the settings defined in [](#opt-services.prometheus.alertmanager-ntfy.settings).
This is useful to avoid putting secrets into the Nix store.
See <https://github.com/alexbakker/alertmanager-ntfy> for more information.
'';
};
};
config = lib.mkIf cfg.enable {
systemd.services.alertmanager-ntfy = {
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = {
User = "alertmanager-ntfy";
Group = "alertmanager-ntfy";
DynamicUser = true;
LoadCredential = lib.imap0 (i: path: "config-${toString i}.yml:${path}") cfg.extraConfigFiles;
ExecStart = "${lib.getExe cfg.package} --configs ${configsArg}";
Restart = "always";
RestartSec = 5;
# Hardening
AmbientCapabilities = "";
CapabilityBoundingSet = [ "" ];
DevicePolicy = "closed";
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
PrivateUsers = true;
ProcSubset = "pid";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [ "AF_INET AF_INET6" ];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
UMask = "0077";
};
};
};
}
+1 -1
View File
@@ -1117,7 +1117,7 @@ in
private-gpt = handleTest ./private-gpt.nix { };
privatebin = runTest ./privatebin.nix;
privoxy = handleTest ./privoxy.nix { };
prometheus = handleTest ./prometheus { };
prometheus = import ./prometheus { inherit runTest; };
prometheus-exporters = handleTest ./prometheus-exporters.nix { };
prosody = handleTest ./xmpp/prosody.nix { };
prosody-mysql = handleTest ./xmpp/prosody-mysql.nix { };
@@ -0,0 +1,99 @@
{ lib, ... }:
let
ports = {
alertmanager-ntfy = 8000;
ntfy-sh = 8001;
alertmanager = 8002;
};
in
{
name = "alertmanager-ntfy";
meta.maintainers = with lib.maintainers; [ defelo ];
nodes.machine = {
services.prometheus.alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
port = ports.alertmanager;
configuration = {
route = {
receiver = "test";
group_by = [ "..." ];
group_wait = "0s";
group_interval = "1s";
repeat_interval = "2h";
};
receivers = [
{
name = "test";
webhook_configs = [ { url = "http://127.0.0.1:${toString ports.alertmanager-ntfy}/hook"; } ];
}
];
};
};
services.prometheus.alertmanager-ntfy = {
enable = true;
settings = {
http.addr = "127.0.0.1:${toString ports.alertmanager-ntfy}";
ntfy = {
baseurl = "http://127.0.0.1:${toString ports.ntfy-sh}";
notification.topic = "alertmanager";
};
};
};
services.ntfy-sh = {
enable = true;
settings = {
listen-http = "127.0.0.1:${toString ports.ntfy-sh}";
base-url = "http://127.0.0.1:${toString ports.ntfy-sh}";
};
};
};
interactive.nodes.machine = {
services.prometheus.alertmanager.listenAddress = lib.mkForce "0.0.0.0";
services.prometheus.alertmanager-ntfy.settings.http.addr =
lib.mkForce "0.0.0.0:${toString ports.alertmanager-ntfy}";
services.ntfy-sh.settings.listen-http = lib.mkForce "0.0.0.0:${toString ports.ntfy-sh}";
networking.firewall.enable = false;
virtualisation.forwardPorts = lib.mapAttrsToList (_: port: {
from = "host";
host = { inherit port; };
guest = { inherit port; };
}) ports;
};
testScript = ''
import json
import time
machine.wait_for_unit("alertmanager.service")
machine.wait_for_unit("alertmanager-ntfy.service")
machine.wait_for_unit("ntfy-sh.service")
machine.wait_for_open_port(${toString ports.alertmanager})
machine.wait_for_open_port(${toString ports.alertmanager-ntfy})
machine.wait_for_open_port(${toString ports.ntfy-sh})
machine.succeed("""curl 127.0.0.1:${toString ports.alertmanager}/api/v2/alerts \
-X POST -H 'Content-Type: application/json' \
-d '[{ \
"labels": {"alertname": "test"},
"annotations": {"summary": "alert summary", "description": "alert description"} \
}]'""")
while not (resp := machine.succeed("curl '127.0.0.1:${toString ports.ntfy-sh}/alertmanager/json?poll=1'")):
time.sleep(1)
msg = json.loads(resp)
assert msg["title"] == "alert summary"
assert msg["message"] == "alert description"
assert msg["priority"] == 4
assert "red_circle" in msg["tags"]
'';
}
+125 -139
View File
@@ -1,160 +1,146 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{ pkgs, ... }:
{
name = "prometheus-alertmanager";
{
name = "prometheus-alertmanager";
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
alertmanagers = [
{
scheme = "http";
static_configs = [
{
targets = [
"alertmanager:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
];
rules = [
''
groups:
- name: test
rules:
- alert: InstanceDown
expr: up == 0
for: 5s
labels:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
''
];
scrapeConfigs = [
{
job_name = "alertmanager";
static_configs = [
{
targets = [
"alertmanager:${toString config.services.prometheus.alertmanager.port}"
];
}
];
}
{
job_name = "node";
static_configs = [
{
targets = [
"node:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
];
};
};
alertmanager =
{ config, pkgs, ... }:
{
services.prometheus.alertmanager = {
enable = true;
openFirewall = true;
configuration = {
global = {
resolve_timeout = "1m";
};
route = {
# Root route node
receiver = "test";
group_by = [ "..." ];
continue = false;
group_wait = "1s";
group_interval = "15s";
repeat_interval = "24h";
};
receivers = [
{
name = "test";
webhook_configs = [
{
url = "http://logger:6725";
send_resolved = true;
max_alerts = 0;
}
];
}
alertmanagers = [
{
scheme = "http";
static_configs = [
{ targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; }
];
}
];
rules = [
''
groups:
- name: test
rules:
- alert: InstanceDown
expr: up == 0
for: 5s
labels:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
''
];
scrapeConfigs = [
{
job_name = "alertmanager";
static_configs = [
{ targets = [ "alertmanager:${toString config.services.prometheus.alertmanager.port}" ]; }
];
}
{
job_name = "node";
static_configs = [
{ targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
};
};
alertmanager =
{ config, pkgs, ... }:
{
services.prometheus.alertmanager = {
enable = true;
openFirewall = true;
configuration = {
global = {
resolve_timeout = "1m";
};
route = {
# Root route node
receiver = "test";
group_by = [ "..." ];
continue = false;
group_wait = "1s";
group_interval = "15s";
repeat_interval = "24h";
};
receivers = [
{
name = "test";
webhook_configs = [
{
url = "http://logger:6725";
send_resolved = true;
max_alerts = 0;
}
];
}
];
};
};
};
logger =
{ config, pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 6725 ];
logger =
{ config, pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 6725 ];
services.prometheus.alertmanagerWebhookLogger.enable = true;
};
};
services.prometheus.alertmanagerWebhookLogger.enable = true;
};
};
testScript = ''
alertmanager.wait_for_unit("alertmanager")
alertmanager.wait_for_open_port(9093)
alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
#alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")
testScript = ''
alertmanager.wait_for_unit("alertmanager")
alertmanager.wait_for_open_port(9093)
alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
#alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")
logger.wait_for_unit("alertmanager-webhook-logger")
logger.wait_for_open_port(6725)
logger.wait_for_unit("alertmanager-webhook-logger")
logger.wait_for_open_port(6725)
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
+ "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
+ "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
)
logger.wait_until_succeeds(
"journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
)
logger.wait_until_succeeds(
"journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
)
logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v ''"))
logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v ''"))
alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v ''"))
'';
}
)
alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v ''"))
'';
}
+91 -103
View File
@@ -1,120 +1,108 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{
name = "prometheus-config-reload";
{
name = "prometheus-config-reload";
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
enableReload = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "prometheus:${toString config.services.prometheus.port}" ]; } ];
}
];
};
services.prometheus = {
enable = true;
enableReload = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
specialisation = {
"prometheus-config-change" = {
configuration = {
environment.systemPackages = [ pkgs.yq ];
# This configuration just adds a new prometheus job
# to scrape the node_exporter metrics of the s3 machine.
services.prometheus = {
scrapeConfigs = [
{
targets = [
"prometheus:${toString config.services.prometheus.port}"
job_name = "node";
static_configs = [
{ targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
}
];
};
specialisation = {
"prometheus-config-change" = {
configuration = {
environment.systemPackages = [ pkgs.yq ];
# This configuration just adds a new prometheus job
# to scrape the node_exporter metrics of the s3 machine.
services.prometheus = {
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ];
}
];
}
];
};
};
};
};
};
};
};
};
testScript = ''
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
testScript = ''
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
# Check if switching to a NixOS configuration that changes the prometheus
# configuration reloads (instead of restarts) prometheus before the switch
# finishes successfully:
with subtest("config change reloads prometheus"):
import json
# We check if prometheus has finished reloading by looking for the message
# "Completed loading of configuration file" in the journal between the start
# and finish of switching to the new NixOS configuration.
#
# To mark the start we record the journal cursor before starting the switch:
cursor_before_switching = json.loads(
prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
)["__CURSOR"]
# Check if switching to a NixOS configuration that changes the prometheus
# configuration reloads (instead of restarts) prometheus before the switch
# finishes successfully:
with subtest("config change reloads prometheus"):
import json
# We check if prometheus has finished reloading by looking for the message
# "Completed loading of configuration file" in the journal between the start
# and finish of switching to the new NixOS configuration.
#
# To mark the start we record the journal cursor before starting the switch:
cursor_before_switching = json.loads(
prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
)["__CURSOR"]
# Now we switch:
prometheus_config_change = prometheus.succeed(
"readlink /run/current-system/specialisation/prometheus-config-change"
).strip()
prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
# Now we switch:
prometheus_config_change = prometheus.succeed(
"readlink /run/current-system/specialisation/prometheus-config-change"
).strip()
prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
# Next we retrieve all logs since the start of switching:
logs_after_starting_switching = prometheus.succeed(
"""
journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
""".format(
cursor_before_switching=cursor_before_switching
)
)
# Finally we check if the message "Completed loading of configuration file"
# occurs before the "finished switching to system configuration" message:
finished_switching_msg = (
"finished switching to system configuration " + prometheus_config_change
)
reloaded_before_switching_finished = False
finished_switching = False
for log_line in logs_after_starting_switching.split("\n"):
msg = json.loads(log_line)["MESSAGE"]
if "Completed loading of configuration file" in msg:
reloaded_before_switching_finished = True
if msg == finished_switching_msg:
finished_switching = True
break
assert reloaded_before_switching_finished
assert finished_switching
# Check if the reloaded config includes the new node job:
prometheus.succeed(
# Next we retrieve all logs since the start of switching:
logs_after_starting_switching = prometheus.succeed(
"""
curl -sf http://127.0.0.1:9090/api/v1/status/config \
| jq -r .data.yaml \
| yq '.scrape_configs | any(.job_name == "node")' \
| grep true
"""
)
'';
}
)
journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
""".format(
cursor_before_switching=cursor_before_switching
)
)
# Finally we check if the message "Completed loading of configuration file"
# occurs before the "finished switching to system configuration" message:
finished_switching_msg = (
"finished switching to system configuration " + prometheus_config_change
)
reloaded_before_switching_finished = False
finished_switching = False
for log_line in logs_after_starting_switching.split("\n"):
msg = json.loads(log_line)["MESSAGE"]
if "Completed loading of configuration file" in msg:
reloaded_before_switching_finished = True
if msg == finished_switching_msg:
finished_switching = True
break
assert reloaded_before_switching_finished
assert finished_switching
# Check if the reloaded config includes the new node job:
prometheus.succeed(
"""
curl -sf http://127.0.0.1:9090/api/v1/status/config \
| jq -r .data.yaml \
| yq '.scrape_configs | any(.job_name == "node")' \
| grep true
"""
)
'';
}
+8 -11
View File
@@ -1,14 +1,11 @@
{
system ? builtins.currentSystem,
config ? { },
pkgs ? import ../../.. { inherit system config; },
}:
{ runTest }:
{
alertmanager = import ./alertmanager.nix { inherit system pkgs; };
config-reload = import ./config-reload.nix { inherit system pkgs; };
federation = import ./federation.nix { inherit system pkgs; };
prometheus-pair = import ./prometheus-pair.nix { inherit system pkgs; };
pushgateway = import ./pushgateway.nix { inherit system pkgs; };
remote-write = import ./remote-write.nix { inherit system pkgs; };
alertmanager = runTest ./alertmanager.nix;
alertmanager-ntfy = runTest ./alertmanager-ntfy.nix;
config-reload = runTest ./config-reload.nix;
federation = runTest ./federation.nix;
prometheus-pair = runTest ./prometheus-pair.nix;
pushgateway = runTest ./pushgateway.nix;
remote-write = runTest ./remote-write.nix;
}
+174 -198
View File
@@ -1,227 +1,203 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{
name = "prometheus-federation";
{
name = "prometheus-federation";
nodes = {
global1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
global1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
];
};
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
};
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
global2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
};
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
}
];
}
];
};
};
prometheus1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
global2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node1:${toString config.services.prometheus.exporters.node.port}"
];
}
scrapeConfigs = [
{
job_name = "federate";
honor_labels = true;
metrics_path = "/federate";
params = {
"match[]" = [
"{job=\"node\"}"
"{job=\"prometheus\"}"
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"global1:${toString config.services.prometheus.port}"
"global2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
prometheus2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
prometheus1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node2:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{ targets = [ "node1:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
{
job_name = "prometheus";
static_configs = [ { targets = [ "prometheus1:${toString config.services.prometheus.port}" ]; } ];
}
];
};
};
node1 =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
prometheus2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{ targets = [ "node2:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
{
job_name = "prometheus";
static_configs = [ { targets = [ "prometheus2:${toString config.services.prometheus.port}" ]; } ];
}
];
};
};
node2 =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
node1 =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
};
testScript = ''
for machine in node1, node2:
machine.wait_for_unit("prometheus-node-exporter")
machine.wait_for_open_port(9100)
node2 =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
};
for machine in prometheus1, prometheus2, global1, global2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
testScript = ''
for machine in node1, node2:
machine.wait_for_unit("prometheus-node-exporter")
machine.wait_for_open_port(9100)
# Verify both servers got the same data from the exporter
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
for machine in prometheus1, prometheus2, global1, global2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
for machine in global1, global2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
# Verify both servers got the same data from the exporter
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"4\"'"
)
'';
}
)
for machine in global1, global2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
+ "jq '.data.result[0].value[1]' | grep '\"4\"'"
)
'';
}
+76 -78
View File
@@ -1,93 +1,91 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{ pkgs, ... }:
{
name = "prometheus-pair";
{
name = "prometheus-pair";
nodes = {
prometheus1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
prometheus1 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
prometheus2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
prometheus2 =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [
{
targets = [
"prometheus1:${toString config.services.prometheus.port}"
"prometheus2:${toString config.services.prometheus.port}"
];
}
];
}
];
};
};
};
};
testScript = ''
for machine in prometheus1, prometheus2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'")
machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy")
testScript = ''
for machine in prometheus1, prometheus2:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'version=${pkgs.prometheus.version}'")
machine.wait_until_succeeds("curl -sSf http://localhost:9090/-/healthy")
# Prometheii ready - run some queries
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
# Prometheii ready - run some queries
for machine in prometheus1, prometheus2:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\",version=\"${pkgs.prometheus.version}\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_build_info\{instance=\"prometheus1:9090\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus.version}\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(prometheus_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].value[1]' | grep '\"2\"'"
)
prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v ''"))
'';
}
)
prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v ''"))
'';
}
+70 -81
View File
@@ -1,102 +1,91 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{ pkgs, ... }:
{
name = "prometheus-pushgateway";
{
name = "prometheus-pushgateway";
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
scrapeConfigs = [
{
job_name = "pushgateway";
static_configs = [
{
targets = [
"pushgateway:9091"
];
}
];
}
];
};
scrapeConfigs = [
{
job_name = "pushgateway";
static_configs = [ { targets = [ "pushgateway:9091" ]; } ];
}
];
};
};
pushgateway =
{ config, pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 9091 ];
pushgateway =
{ config, pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 9091 ];
services.prometheus.pushgateway = {
enable = true;
};
services.prometheus.pushgateway = {
enable = true;
};
};
client =
{ config, pkgs, ... }:
{
};
};
client = { config, pkgs, ... }: { };
};
testScript = ''
pushgateway.wait_for_unit("pushgateway")
pushgateway.wait_for_open_port(9091)
pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready")
pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'")
testScript = ''
pushgateway.wait_for_unit("pushgateway")
pushgateway.wait_for_open_port(9091)
pushgateway.wait_until_succeeds("curl -s http://127.0.0.1:9091/-/ready")
pushgateway.wait_until_succeeds("journalctl -o cat -u pushgateway.service | grep 'version=${pkgs.prometheus-pushgateway.version}'")
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_for_unit("prometheus")
prometheus.wait_for_open_port(9090)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"pushgateway\"\})' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(pushgateway_build_info)%20by%20(version)' | "
+ "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-pushgateway.version}\"'"
)
# Add a metric and check in Prometheus
client.wait_until_succeeds(
"echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job"
)
# Add a metric and check in Prometheus
client.wait_until_succeeds(
"echo 'some_metric 3.14' | curl --data-binary @- http://pushgateway:9091/metrics/job/some_job"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep 'null'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep 'null'"
)
# Delete the metric, check not in Prometheus
client.wait_until_succeeds(
"curl -X DELETE http://pushgateway:9091/metrics/job/some_job"
)
# Delete the metric, check not in Prometheus
client.wait_until_succeeds(
"curl -X DELETE http://pushgateway:9091/metrics/job/some_job"
)
prometheus.wait_until_fails(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_fails(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' | "
+ "jq '.data.result[0].value[1]' | grep '\"3.14\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
prometheus.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=absent(some_metric)' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v ''"))
'';
}
)
pushgateway.log(pushgateway.succeed("systemd-analyze security pushgateway.service | grep -v ''"))
'';
}
+53 -65
View File
@@ -1,81 +1,69 @@
import ../make-test-python.nix (
{ lib, pkgs, ... }:
{
name = "prometheus-remote-write";
{
name = "prometheus-remote-write";
nodes = {
receiver =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
nodes = {
receiver =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
extraFlags = [ "--web.enable-remote-write-receiver" ];
};
extraFlags = [ "--web.enable-remote-write-receiver" ];
};
};
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
prometheus =
{ config, pkgs, ... }:
{
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
services.prometheus = {
enable = true;
globalConfig.scrape_interval = "2s";
remoteWrite = [
{
url = "http://receiver:9090/api/v1/write";
}
];
remoteWrite = [ { url = "http://receiver:9090/api/v1/write"; } ];
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [
"node:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
];
};
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{ targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
];
};
};
node =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
node =
{ config, pkgs, ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
};
};
};
};
testScript = ''
node.wait_for_unit("prometheus-node-exporter")
node.wait_for_open_port(9100)
testScript = ''
node.wait_for_unit("prometheus-node-exporter")
node.wait_for_open_port(9100)
for machine in prometheus, receiver:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
for machine in prometheus, receiver:
machine.wait_for_unit("prometheus")
machine.wait_for_open_port(9090)
# Verify both servers got the same data from the exporter
for machine in prometheus, receiver:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
'';
}
)
# Verify both servers got the same data from the exporter
for machine in prometheus, receiver:
machine.wait_until_succeeds(
"curl -sf 'http://127.0.0.1:9090/api/v1/query?query=node_exporter_build_info\{instance=\"node:9100\"\}' | "
+ "jq '.data.result[0].value[1]' | grep '\"1\"'"
)
'';
}
@@ -0,0 +1,43 @@
{
lib,
buildGoModule,
fetchFromGitHub,
nixosTests,
nix-update-script,
}:
buildGoModule {
pname = "alertmanager-ntfy";
version = "0-unstable-2025-02-24";
src = fetchFromGitHub {
owner = "alexbakker";
repo = "alertmanager-ntfy";
rev = "4573b96077faf39c3d04df913e93d9ded1f1a16c";
hash = "sha256-JmXeDZBcbRDEaDVt7HuR9L9WZzrtqDrUMpHM7cHSQO0=";
};
vendorHash = "sha256-e1JAoDNm2+xB/bZcEGr5l4+va8GIg1R8pdj3d+/Y+UY=";
doInstallCheck = true;
installCheckPhase = ''
runHook preInstallCheck
$out/bin/alertmanager-ntfy --help > /dev/null
runHook postInstallCheck
'';
passthru = {
tests = { inherit (nixosTests.prometheus) alertmanager-ntfy; };
updateScript = nix-update-script { extraArgs = [ "--version=branch=master" ]; };
};
meta = {
description = "Forwards Prometheus Alertmanager notifications to ntfy.sh";
homepage = "https://github.com/alexbakker/alertmanager-ntfy";
license = lib.licenses.gpl3Only;
maintainers = with lib.maintainers; [ defelo ];
mainProgram = "alertmanager-ntfy";
};
}