From 137d1e1830d8955ad04c4fdeb4288bf288d55933 Mon Sep 17 00:00:00 2001 From: Wael Nasreddine Date: Thu, 8 Jan 2026 19:11:24 -0800 Subject: [PATCH] nixos/tests/ncps: Implement an HA test --- nixos/tests/all-tests.nix | 2 +- nixos/tests/ncps-ha.nix | 210 ++++++++++++++++++++++++++++++++ nixos/tests/ncps-storage-s3.nix | 142 --------------------- 3 files changed, 211 insertions(+), 143 deletions(-) create mode 100644 nixos/tests/ncps-ha.nix delete mode 100644 nixos/tests/ncps-storage-s3.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index a17ceeb311e5..3c0547d4578c 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1037,7 +1037,7 @@ in imports = [ ./ncps.nix ]; defaults.services.ncps.cache.storage.local = "/path/to/ncps"; }; - ncps-storage-s3 = runTest ./ncps-storage-s3.nix; + ncps-ha = runTest ./ncps-ha.nix; ndppd = runTest ./ndppd.nix; nebula-lighthouse-service = runTest ./nebula-lighthouse-service.nix; nebula.connectivity = runTest ./nebula/connectivity.nix; diff --git a/nixos/tests/ncps-ha.nix b/nixos/tests/ncps-ha.nix new file mode 100644 index 000000000000..c39962c0536c --- /dev/null +++ b/nixos/tests/ncps-ha.nix @@ -0,0 +1,210 @@ +{ + lib, + pkgs, + ... +}: +let + # s3 creds + bucket = "ncps"; + region = "us-west-1"; + accessKey = builtins.toFile "minio-access-key" "easy-key"; + secretKey = builtins.toFile "minio-secret-key" "easy-secret"; + + # pg creds + postgresPassword = "easypwd"; + + # redis creds + redisPassword = "easypwd"; + + initMinio = pkgs.writeShellScriptBin "init-minio.sh" '' + set -euo pipefail + + mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin + mc mb local/${bucket} + mc admin user svcacct add --access-key "$(cat ${accessKey})" --secret-key "$(cat ${secretKey})" local minioadmin + ''; + + ncpsAttrs = hostname: { + services.ncps = { + enable = true; + + analytics.reporting.enable = false; + + cache = { + hostName = hostname; + + databaseURL = "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable"; + + secretKeyPath = builtins.toString ( + pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA==" + ); + + redis = { + addresses = [ "redis:6379" ]; + passwordFile = builtins.toFile "redis-password" redisPassword; + }; + + storage.s3 = { + inherit bucket region; + + endpoint = "http://minio:9000"; + + accessKeyIdPath = accessKey; + secretAccessKeyPath = secretKey; + }; + + upstream = { + urls = [ "http://harmonia:5000" ]; + publicKeys = [ + "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo=" + ]; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ 8501 ]; + }; +in +{ + name = "ncps-storage-s3"; + meta = with lib.maintainers; { + maintainers = [ + aciceri + kalbasit + ]; + }; + + nodes = { + client0 = { + nix.settings = { + substituters = lib.mkForce [ "http://ncps0:8501" ]; + trusted-public-keys = lib.mkForce [ + "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg=" + ]; + }; + }; + + client1 = { + nix.settings = { + substituters = lib.mkForce [ "http://ncps1:8501" ]; + trusted-public-keys = lib.mkForce [ + "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg=" + ]; + }; + }; + + harmonia = { + services.harmonia = { + enable = true; + signKeyPaths = [ + (pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==") + ]; + settings.priority = 35; + }; + + networking.firewall.allowedTCPPorts = [ 5000 ]; + system.extraDependencies = [ pkgs.emptyFile ]; + }; + + minio = { + services.minio = { + inherit region; + + enable = true; + }; + + networking.firewall.allowedTCPPorts = [ 9000 ]; + environment.systemPackages = [ + pkgs.minio-client + initMinio + ]; + }; + + ncps0 = ncpsAttrs "ncps0"; + ncps1 = ncpsAttrs "ncps1"; + + postgres = { + services.postgresql = { + enable = true; + enableTCPIP = true; + authentication = '' + host all all all scram-sha-256 + ''; + initialScript = pkgs.writeText "init-postgres.sql" '' + CREATE DATABASE "ncps" WITH ENCODING = 'UTF8'; + CREATE ROLE "ncps" WITH LOGIN PASSWORD '${ + builtins.replaceStrings [ "'" ] [ "''" ] postgresPassword + }'; + ALTER DATABASE "ncps" OWNER TO "ncps"; + ''; + }; + + networking.firewall.allowedTCPPorts = [ 5432 ]; + }; + + redis = { + services.redis.servers.ncps = { + enable = true; + openFirewall = true; + port = 6379; + requirePass = redisPassword; + bind = null; + }; + }; + }; + + testScript = + { nodes, ... }: + let + narinfoName = + (lib.strings.removePrefix "/nix/store/" ( + lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath + )) + + ".narinfo"; + + narinfoNameChars = lib.strings.stringToCharacters narinfoName; + + narinfoPath = lib.concatStringsSep "/" [ + (builtins.head nodes.minio.services.minio.dataDir) + bucket + "store/narinfo" + (lib.lists.elemAt narinfoNameChars 0) + ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1)) + narinfoName + "xl.meta" + ]; + in + '' + harmonia.start() + minio.start() + postgres.start() + redis.start() + + minio.wait_for_unit("minio.service") + + minio.wait_until_succeeds("init-minio.sh") + + postgres.wait_for_unit("postgresql.service") + redis.wait_for_unit("redis-ncps.service") + + redis.wait_until_succeeds("redis-cli -h redis -p 6379 -a '${redisPassword}' ping") + + start_all() + + harmonia.wait_for_unit("harmonia.service") + + ncps0.wait_for_unit("ncps.service") + ncps1.wait_for_unit("ncps.service") + + client0.wait_until_succeeds("curl -f http://ncps0:8501/ | grep '\"hostname\":\"${toString nodes.ncps0.services.ncps.cache.hostName}\"' >&2") + client1.wait_until_succeeds("curl -f http://ncps1:8501/ | grep '\"hostname\":\"${toString nodes.ncps1.services.ncps.cache.hostName}\"' >&2") + + client0.succeed("cat /etc/nix/nix.conf >&2") + client0.succeed("nix-store --realise ${pkgs.emptyFile}") + + client1.succeed("cat /etc/nix/nix.conf >&2") + client1.succeed("nix-store --realise ${pkgs.emptyFile}") + + minio.succeed("cat ${narinfoPath} >&2") + ''; +} diff --git a/nixos/tests/ncps-storage-s3.nix b/nixos/tests/ncps-storage-s3.nix deleted file mode 100644 index c75557114a1b..000000000000 --- a/nixos/tests/ncps-storage-s3.nix +++ /dev/null @@ -1,142 +0,0 @@ -{ - lib, - pkgs, - ... -}: -let - bucket = "ncps"; - region = "us-west-1"; - accessKey = "test-access-key"; - secretKey = "test-secret-key"; - - initMinio = pkgs.writeShellScriptBin "init-minio.sh" '' - set -euo pipefail - - mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin - mc mb local/${bucket} - mc admin user svcacct add --access-key ${accessKey} --secret-key ${secretKey} local minioadmin - ''; -in -{ - name = "ncps-storage-s3"; - meta = with lib.maintainers; { - maintainers = [ - aciceri - kalbasit - ]; - }; - - nodes = { - harmonia = { - services.harmonia = { - enable = true; - signKeyPaths = [ - (pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==") - ]; - settings.priority = 35; - }; - - networking.firewall.allowedTCPPorts = [ 5000 ]; - system.extraDependencies = [ pkgs.emptyFile ]; - }; - - minio = { - services.minio = { - inherit region; - - enable = true; - }; - - networking.firewall.allowedTCPPorts = [ 9000 ]; - environment.systemPackages = [ - pkgs.minio-client - initMinio - ]; - }; - - ncps = { - services.ncps = { - enable = true; - - analytics.reporting.enable = false; - - cache = { - hostName = "ncps"; - - secretKeyPath = builtins.toString ( - pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA==" - ); - - storage.s3 = { - inherit bucket region; - - endpoint = "http://minio:9000"; - - accessKeyIdPath = builtins.toFile "minio-access-key" accessKey; - secretAccessKeyPath = builtins.toFile "minio-secret-key" secretKey; - }; - - upstream = { - urls = [ "http://harmonia:5000" ]; - publicKeys = [ - "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo=" - ]; - }; - }; - }; - - networking.firewall.allowedTCPPorts = [ 8501 ]; - }; - - client = { - nix.settings = { - substituters = lib.mkForce [ "http://ncps:8501" ]; - trusted-public-keys = lib.mkForce [ - "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg=" - ]; - }; - }; - }; - - testScript = - { nodes, ... }: - let - narinfoName = - (lib.strings.removePrefix "/nix/store/" ( - lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath - )) - + ".narinfo"; - - narinfoNameChars = lib.strings.stringToCharacters narinfoName; - - narinfoPath = lib.concatStringsSep "/" [ - (builtins.head nodes.minio.services.minio.dataDir) - bucket - "store/narinfo" - (lib.lists.elemAt narinfoNameChars 0) - ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1)) - narinfoName - "xl.meta" - ]; - in - '' - minio.start() - - minio.wait_for_unit("minio.service") - - minio.wait_until_succeeds("init-minio.sh") - - start_all() - - harmonia.wait_for_unit("harmonia.service") - - ncps.wait_for_unit("ncps.service") - - client.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2") - - client.succeed("cat /etc/nix/nix.conf >&2") - client.succeed("nix-store --realise ${pkgs.emptyFile}") - - minio.succeed("cat ${narinfoPath} >&2") - ''; -}