diff --git a/nixos/modules/services/monitoring/grafana-to-ntfy.nix b/nixos/modules/services/monitoring/grafana-to-ntfy.nix index db35792363b8..09512e1a37e0 100644 --- a/nixos/modules/services/monitoring/grafana-to-ntfy.nix +++ b/nixos/modules/services/monitoring/grafana-to-ntfy.nix @@ -9,9 +9,11 @@ let cfg = config.services.grafana-to-ntfy; in { + meta.maintainers = with lib.maintainers; [ kittyandrew ]; + options = { services.grafana-to-ntfy = { - enable = lib.mkEnableOption "Grafana-to-ntfy (ntfy.sh) alerts channel"; + enable = lib.mkEnableOption "grafana-to-ntfy, a Grafana/Alertmanager to ntfy.sh bridge"; package = lib.mkPackageOption pkgs "grafana-to-ntfy" { }; @@ -33,64 +35,124 @@ in }; ntfyBAuthPass = lib.mkOption { - type = lib.types.path; + type = lib.types.nullOr lib.types.path; description = '' The path to the password for the specified ntfy-sh user. Setting this option is required when using a ntfy-sh instance with access control enabled. ''; default = null; + example = "/run/secrets/grafana-to-ntfy-ntfy-pass"; }; bauthUser = lib.mkOption { - type = lib.types.str; + type = lib.types.nullOr lib.types.str; description = '' - The user that you will authenticate with in the Grafana webhook settings. - You can set this to whatever you like, as this is not the same as the ntfy-sh user. + The user for Basic Auth on incoming webhook requests from Grafana or Alertmanager. + When set together with {option}`bauthPass`, incoming requests require Basic Auth. + When both are null, the endpoint is open (unauthenticated). ''; - default = "admin"; + default = null; + example = "admin"; }; bauthPass = lib.mkOption { - type = lib.types.path; - description = "The path to the password you will use in the Grafana webhook settings."; + type = lib.types.nullOr lib.types.path; + description = '' + Path to the password file for Basic Auth on incoming webhook requests. + When set together with {option}`bauthUser`, incoming requests require Basic Auth. + When both are null, the endpoint is open (unauthenticated). + ''; + default = null; + example = "/run/secrets/grafana-to-ntfy-bauth-pass"; + }; + + markdown = lib.mkOption { + type = lib.types.bool; + description = "Enable Markdown formatting in ntfy notifications. Sets the X-Markdown header."; + default = false; + }; + + port = lib.mkOption { + type = lib.types.port; + description = "Port to listen on."; + default = 8080; + }; + + address = lib.mkOption { + type = lib.types.str; + description = "Address to listen on."; + default = "127.0.0.1"; + example = "0.0.0.0"; }; }; }; }; config = lib.mkIf cfg.enable { - systemd.services.grafana-to-ntfy = { - wantedBy = [ "multi-user.target" ]; - - script = '' - export BAUTH_PASS=$(${lib.getExe' config.systemd.package "systemd-creds"} cat BAUTH_PASS_FILE) - ${lib.optionalString (cfg.settings.ntfyBAuthPass != null) '' - export NTFY_BAUTH_PASS=$(${lib.getExe' config.systemd.package "systemd-creds"} cat NTFY_BAUTH_PASS_FILE) - ''} - exec ${lib.getExe cfg.package} - ''; - - environment = { - NTFY_URL = cfg.settings.ntfyUrl; - BAUTH_USER = cfg.settings.bauthUser; + assertions = [ + { + assertion = (cfg.settings.bauthUser == null) == (cfg.settings.bauthPass == null); + message = "services.grafana-to-ntfy: bauthUser and bauthPass must both be set or both be null"; } - // lib.optionalAttrs (cfg.settings.ntfyBAuthUser != null) { - NTFY_BAUTH_USER = cfg.settings.ntfyBAuthUser; - }; + { + assertion = (cfg.settings.ntfyBAuthUser == null) == (cfg.settings.ntfyBAuthPass == null); + message = "services.grafana-to-ntfy: ntfyBAuthUser and ntfyBAuthPass must both be set or both be null"; + } + ]; + + systemd.services.grafana-to-ntfy = { + description = "Grafana/Alertmanager to ntfy.sh bridge"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + script = + let + optionalCred = name: envVar: '' + export ${envVar}="$(${lib.getExe' config.systemd.package "systemd-creds"} cat ${name})" + ''; + in + '' + ${lib.optionalString (cfg.settings.bauthPass != null) (optionalCred "BAUTH_PASS_FILE" "BAUTH_PASS")} + ${lib.optionalString (cfg.settings.ntfyBAuthPass != null) (optionalCred "NTFY_BAUTH_PASS_FILE" "NTFY_BAUTH_PASS")} + exec ${lib.getExe cfg.package} + ''; + + environment = + { + NTFY_URL = cfg.settings.ntfyUrl; + ROCKET_PORT = toString cfg.settings.port; + ROCKET_ADDRESS = cfg.settings.address; + } + // lib.optionalAttrs (cfg.settings.bauthUser != null) { + BAUTH_USER = cfg.settings.bauthUser; + } + // lib.optionalAttrs (cfg.settings.ntfyBAuthUser != null) { + NTFY_BAUTH_USER = cfg.settings.ntfyBAuthUser; + } + // lib.optionalAttrs cfg.settings.markdown { + MARKDOWN = "true"; + }; serviceConfig = { - LoadCredential = [ - "BAUTH_PASS_FILE:${cfg.settings.bauthPass}" - ] - ++ lib.optional ( - cfg.settings.ntfyBAuthPass != null - ) "NTFY_BAUTH_PASS_FILE:${cfg.settings.ntfyBAuthPass}"; + LoadCredential = + lib.optional (cfg.settings.bauthPass != null) "BAUTH_PASS_FILE:${cfg.settings.bauthPass}" + ++ lib.optional (cfg.settings.ntfyBAuthPass != null) "NTFY_BAUTH_PASS_FILE:${cfg.settings.ntfyBAuthPass}"; DynamicUser = true; + + Restart = "always"; + RestartSec = 5; + + # Hardening + AmbientCapabilities = [ "" ]; CapabilityBoundingSet = [ "" ]; - DeviceAllow = ""; + DevicePolicy = "closed"; LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; PrivateDevices = true; + PrivateTmp = true; PrivateUsers = true; ProcSubset = "pid"; ProtectClock = true; @@ -101,6 +163,8 @@ in ProtectKernelModules = true; ProtectKernelTunables = true; ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" @@ -108,11 +172,12 @@ in ]; RestrictNamespaces = true; RestrictRealtime = true; - MemoryDenyWriteExecute = true; + RestrictSUIDSGID = true; SystemCallArchitectures = "native"; SystemCallFilter = [ "@system-service" "~@privileged" + "~@resources" ]; UMask = "0077"; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 638afcb5f721..0acdf3c6cddd 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -674,6 +674,7 @@ in gotosocial = runTest ./web-apps/gotosocial.nix; goupile = runTest ./web-apps/goupile; grafana = handleTest ./grafana { }; + grafana-to-ntfy = runTest ./grafana-to-ntfy.nix; graphite = runTest ./graphite.nix; grav = runTest ./web-apps/grav.nix; graylog = runTest ./graylog.nix; diff --git a/nixos/tests/grafana-to-ntfy.nix b/nixos/tests/grafana-to-ntfy.nix new file mode 100644 index 000000000000..d7e942296e3a --- /dev/null +++ b/nixos/tests/grafana-to-ntfy.nix @@ -0,0 +1,166 @@ +{ lib, ... }: + +let + ports = { + grafana-to-ntfy = 8080; + ntfy-sh = 8081; + grafana = 3000; + alertmanager = 9093; + }; + ntfyTopic = "grafana-alerts"; +in + +{ + name = "grafana-to-ntfy"; + meta.maintainers = with lib.maintainers; [ kittyandrew ]; + + nodes.machine = { + services.grafana-to-ntfy = { + enable = true; + settings = { + ntfyUrl = "http://127.0.0.1:${toString ports.ntfy-sh}/${ntfyTopic}"; + port = ports.grafana-to-ntfy; + address = "127.0.0.1"; + }; + }; + + services.ntfy-sh = { + enable = true; + settings = { + listen-http = "127.0.0.1:${toString ports.ntfy-sh}"; + base-url = "http://127.0.0.1:${toString ports.ntfy-sh}"; + }; + }; + + services.grafana = { + enable = true; + settings = { + server.http_port = ports.grafana; + server.http_addr = "127.0.0.1"; + security.admin_user = "admin"; + security.admin_password = "admin"; + security.secret_key = "test-only-dummy-key"; + }; + provision.alerting = { + contactPoints.settings = { + apiVersion = 1; + contactPoints = [ + { + orgId = 1; + name = "grafana-to-ntfy"; + receivers = [ + { + uid = "cp_webhook"; + type = "webhook"; + disableResolveMessage = false; + settings = { + url = "http://127.0.0.1:${toString ports.grafana-to-ntfy}"; + httpMethod = "POST"; + }; + } + ]; + } + ]; + }; + policies.settings = { + apiVersion = 1; + policies = [ + { + orgId = 1; + receiver = "grafana-to-ntfy"; + group_by = [ "..." ]; + group_wait = "0s"; + group_interval = "1s"; + repeat_interval = "1h"; + } + ]; + }; + }; + }; + + services.prometheus.alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + port = ports.alertmanager; + configuration = { + route = { + receiver = "grafana-to-ntfy"; + group_by = [ "..." ]; + group_wait = "0s"; + group_interval = "1s"; + repeat_interval = "2h"; + }; + receivers = [ + { + name = "grafana-to-ntfy"; + webhook_configs = [ + { url = "http://127.0.0.1:${toString ports.grafana-to-ntfy}"; } + ]; + } + ]; + }; + }; + }; + + interactive.nodes.machine = { + services.grafana-to-ntfy.settings.address = lib.mkForce "0.0.0.0"; + services.grafana.settings.server.http_addr = lib.mkForce "0.0.0.0"; + services.prometheus.alertmanager.listenAddress = lib.mkForce "0.0.0.0"; + services.ntfy-sh.settings.listen-http = lib.mkForce "0.0.0.0:${toString ports.ntfy-sh}"; + networking.firewall.enable = false; + virtualisation.forwardPorts = lib.mapAttrsToList (_: port: { + from = "host"; + host = { inherit port; }; + guest = { inherit port; }; + }) ports; + }; + + testScript = '' + import json + + machine.wait_for_unit("grafana-to-ntfy.service") + machine.wait_for_unit("ntfy-sh.service") + machine.wait_for_unit("grafana.service") + machine.wait_for_unit("alertmanager.service") + machine.wait_for_open_port(${toString ports.grafana-to-ntfy}) + machine.wait_for_open_port(${toString ports.ntfy-sh}) + machine.wait_for_open_port(${toString ports.grafana}) + machine.wait_for_open_port(${toString ports.alertmanager}) + + with subtest("Health endpoint returns 200"): + machine.succeed("curl -sf http://127.0.0.1:${toString ports.grafana-to-ntfy}/health") + + with subtest("Alertmanager alert arrives at ntfy"): + machine.succeed( + "curl -sf http://127.0.0.1:${toString ports.alertmanager}/api/v2/alerts" + " -X POST -H 'Content-Type: application/json'" + " -d '[{\"labels\": {\"alertname\": \"TestAlertFromAM\"}}]'" + ) + # grep makes wait_until_succeeds retry: ntfy returns 200 with empty body when no messages exist + resp = machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:${toString ports.ntfy-sh}/${ntfyTopic}/json?poll=1'" + " | grep '\"title\":\"Alertmanager\"'" + ) + msg = json.loads(resp.strip()) + assert msg["title"] == "Alertmanager", f"Expected title 'Alertmanager', got '{msg['title']}'" + assert "warning" in msg["tags"], f"Expected 'warning' in tags, got {msg['tags']}" + assert "firing" in msg["tags"], f"Expected 'firing' in tags, got {msg['tags']}" + + with subtest("Grafana alert arrives at ntfy"): + machine.succeed( + "curl -sf http://127.0.0.1:${toString ports.grafana}/api/alertmanager/grafana/config/api/v1/receivers/test" + " -u admin:admin" + " -X POST -H 'Content-Type: application/json'" + """ -d '{"receivers": [{"name": "grafana-to-ntfy", "grafana_managed_receiver_configs": [{"uid": "cp_webhook", "name": "webhook", "type": "webhook", "disableResolveMessage": false, "settings": {"url": "http://127.0.0.1:${toString ports.grafana-to-ntfy}", "httpMethod": "POST"}}]}]}'""" + ) + # grep ensures we wait for the Grafana message specifically (see above) + resp = machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:${toString ports.ntfy-sh}/${ntfyTopic}/json?poll=1'" + " | grep 'FIRING'" + ) + msg = json.loads(resp.strip()) + assert "[FIRING:1]" in msg["title"], f"Expected Grafana title with '[FIRING:1]', got '{msg['title']}'" + assert "warning" in msg["tags"], f"Expected 'warning' in tags, got {msg['tags']}" + assert "firing" in msg["tags"], f"Expected 'firing' in tags, got {msg['tags']}" + ''; +}