nixos/tests/ncps: Implement an HA test
This commit is contained in:
Wael Nasreddine
@@ -1037,7 +1037,7 @@ in
|
||||
imports = [ ./ncps.nix ];
|
||||
defaults.services.ncps.cache.storage.local = "/path/to/ncps";
|
||||
};
|
||||
ncps-storage-s3 = runTest ./ncps-storage-s3.nix;
|
||||
ncps-ha = runTest ./ncps-ha.nix;
|
||||
ndppd = runTest ./ndppd.nix;
|
||||
nebula-lighthouse-service = runTest ./nebula-lighthouse-service.nix;
|
||||
nebula.connectivity = runTest ./nebula/connectivity.nix;
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
# s3 creds
|
||||
bucket = "ncps";
|
||||
region = "us-west-1";
|
||||
accessKey = builtins.toFile "minio-access-key" "easy-key";
|
||||
secretKey = builtins.toFile "minio-secret-key" "easy-secret";
|
||||
|
||||
# pg creds
|
||||
postgresPassword = "easypwd";
|
||||
|
||||
# redis creds
|
||||
redisPassword = "easypwd";
|
||||
|
||||
initMinio = pkgs.writeShellScriptBin "init-minio.sh" ''
|
||||
set -euo pipefail
|
||||
|
||||
mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin
|
||||
mc mb local/${bucket}
|
||||
mc admin user svcacct add --access-key "$(cat ${accessKey})" --secret-key "$(cat ${secretKey})" local minioadmin
|
||||
'';
|
||||
|
||||
ncpsAttrs = hostname: {
|
||||
services.ncps = {
|
||||
enable = true;
|
||||
|
||||
analytics.reporting.enable = false;
|
||||
|
||||
cache = {
|
||||
hostName = hostname;
|
||||
|
||||
databaseURL = "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable";
|
||||
|
||||
secretKeyPath = builtins.toString (
|
||||
pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA=="
|
||||
);
|
||||
|
||||
redis = {
|
||||
addresses = [ "redis:6379" ];
|
||||
passwordFile = builtins.toFile "redis-password" redisPassword;
|
||||
};
|
||||
|
||||
storage.s3 = {
|
||||
inherit bucket region;
|
||||
|
||||
endpoint = "http://minio:9000";
|
||||
|
||||
accessKeyIdPath = accessKey;
|
||||
secretAccessKeyPath = secretKey;
|
||||
};
|
||||
|
||||
upstream = {
|
||||
urls = [ "http://harmonia:5000" ];
|
||||
publicKeys = [
|
||||
"cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 8501 ];
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "ncps-storage-s3";
|
||||
meta = with lib.maintainers; {
|
||||
maintainers = [
|
||||
aciceri
|
||||
kalbasit
|
||||
];
|
||||
};
|
||||
|
||||
nodes = {
|
||||
client0 = {
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://ncps0:8501" ];
|
||||
trusted-public-keys = lib.mkForce [
|
||||
"ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
client1 = {
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://ncps1:8501" ];
|
||||
trusted-public-keys = lib.mkForce [
|
||||
"ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
harmonia = {
|
||||
services.harmonia = {
|
||||
enable = true;
|
||||
signKeyPaths = [
|
||||
(pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==")
|
||||
];
|
||||
settings.priority = 35;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 5000 ];
|
||||
system.extraDependencies = [ pkgs.emptyFile ];
|
||||
};
|
||||
|
||||
minio = {
|
||||
services.minio = {
|
||||
inherit region;
|
||||
|
||||
enable = true;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 9000 ];
|
||||
environment.systemPackages = [
|
||||
pkgs.minio-client
|
||||
initMinio
|
||||
];
|
||||
};
|
||||
|
||||
ncps0 = ncpsAttrs "ncps0";
|
||||
ncps1 = ncpsAttrs "ncps1";
|
||||
|
||||
postgres = {
|
||||
services.postgresql = {
|
||||
enable = true;
|
||||
enableTCPIP = true;
|
||||
authentication = ''
|
||||
host all all all scram-sha-256
|
||||
'';
|
||||
initialScript = pkgs.writeText "init-postgres.sql" ''
|
||||
CREATE DATABASE "ncps" WITH ENCODING = 'UTF8';
|
||||
CREATE ROLE "ncps" WITH LOGIN PASSWORD '${
|
||||
builtins.replaceStrings [ "'" ] [ "''" ] postgresPassword
|
||||
}';
|
||||
ALTER DATABASE "ncps" OWNER TO "ncps";
|
||||
'';
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 5432 ];
|
||||
};
|
||||
|
||||
redis = {
|
||||
services.redis.servers.ncps = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
port = 6379;
|
||||
requirePass = redisPassword;
|
||||
bind = null;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
let
|
||||
narinfoName =
|
||||
(lib.strings.removePrefix "/nix/store/" (
|
||||
lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath
|
||||
))
|
||||
+ ".narinfo";
|
||||
|
||||
narinfoNameChars = lib.strings.stringToCharacters narinfoName;
|
||||
|
||||
narinfoPath = lib.concatStringsSep "/" [
|
||||
(builtins.head nodes.minio.services.minio.dataDir)
|
||||
bucket
|
||||
"store/narinfo"
|
||||
(lib.lists.elemAt narinfoNameChars 0)
|
||||
((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1))
|
||||
narinfoName
|
||||
"xl.meta"
|
||||
];
|
||||
in
|
||||
''
|
||||
harmonia.start()
|
||||
minio.start()
|
||||
postgres.start()
|
||||
redis.start()
|
||||
|
||||
minio.wait_for_unit("minio.service")
|
||||
|
||||
minio.wait_until_succeeds("init-minio.sh")
|
||||
|
||||
postgres.wait_for_unit("postgresql.service")
|
||||
redis.wait_for_unit("redis-ncps.service")
|
||||
|
||||
redis.wait_until_succeeds("redis-cli -h redis -p 6379 -a '${redisPassword}' ping")
|
||||
|
||||
start_all()
|
||||
|
||||
harmonia.wait_for_unit("harmonia.service")
|
||||
|
||||
ncps0.wait_for_unit("ncps.service")
|
||||
ncps1.wait_for_unit("ncps.service")
|
||||
|
||||
client0.wait_until_succeeds("curl -f http://ncps0:8501/ | grep '\"hostname\":\"${toString nodes.ncps0.services.ncps.cache.hostName}\"' >&2")
|
||||
client1.wait_until_succeeds("curl -f http://ncps1:8501/ | grep '\"hostname\":\"${toString nodes.ncps1.services.ncps.cache.hostName}\"' >&2")
|
||||
|
||||
client0.succeed("cat /etc/nix/nix.conf >&2")
|
||||
client0.succeed("nix-store --realise ${pkgs.emptyFile}")
|
||||
|
||||
client1.succeed("cat /etc/nix/nix.conf >&2")
|
||||
client1.succeed("nix-store --realise ${pkgs.emptyFile}")
|
||||
|
||||
minio.succeed("cat ${narinfoPath} >&2")
|
||||
'';
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
bucket = "ncps";
|
||||
region = "us-west-1";
|
||||
accessKey = "test-access-key";
|
||||
secretKey = "test-secret-key";
|
||||
|
||||
initMinio = pkgs.writeShellScriptBin "init-minio.sh" ''
|
||||
set -euo pipefail
|
||||
|
||||
mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin
|
||||
mc mb local/${bucket}
|
||||
mc admin user svcacct add --access-key ${accessKey} --secret-key ${secretKey} local minioadmin
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "ncps-storage-s3";
|
||||
meta = with lib.maintainers; {
|
||||
maintainers = [
|
||||
aciceri
|
||||
kalbasit
|
||||
];
|
||||
};
|
||||
|
||||
nodes = {
|
||||
harmonia = {
|
||||
services.harmonia = {
|
||||
enable = true;
|
||||
signKeyPaths = [
|
||||
(pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==")
|
||||
];
|
||||
settings.priority = 35;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 5000 ];
|
||||
system.extraDependencies = [ pkgs.emptyFile ];
|
||||
};
|
||||
|
||||
minio = {
|
||||
services.minio = {
|
||||
inherit region;
|
||||
|
||||
enable = true;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 9000 ];
|
||||
environment.systemPackages = [
|
||||
pkgs.minio-client
|
||||
initMinio
|
||||
];
|
||||
};
|
||||
|
||||
ncps = {
|
||||
services.ncps = {
|
||||
enable = true;
|
||||
|
||||
analytics.reporting.enable = false;
|
||||
|
||||
cache = {
|
||||
hostName = "ncps";
|
||||
|
||||
secretKeyPath = builtins.toString (
|
||||
pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA=="
|
||||
);
|
||||
|
||||
storage.s3 = {
|
||||
inherit bucket region;
|
||||
|
||||
endpoint = "http://minio:9000";
|
||||
|
||||
accessKeyIdPath = builtins.toFile "minio-access-key" accessKey;
|
||||
secretAccessKeyPath = builtins.toFile "minio-secret-key" secretKey;
|
||||
};
|
||||
|
||||
upstream = {
|
||||
urls = [ "http://harmonia:5000" ];
|
||||
publicKeys = [
|
||||
"cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 8501 ];
|
||||
};
|
||||
|
||||
client = {
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://ncps:8501" ];
|
||||
trusted-public-keys = lib.mkForce [
|
||||
"ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
let
|
||||
narinfoName =
|
||||
(lib.strings.removePrefix "/nix/store/" (
|
||||
lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath
|
||||
))
|
||||
+ ".narinfo";
|
||||
|
||||
narinfoNameChars = lib.strings.stringToCharacters narinfoName;
|
||||
|
||||
narinfoPath = lib.concatStringsSep "/" [
|
||||
(builtins.head nodes.minio.services.minio.dataDir)
|
||||
bucket
|
||||
"store/narinfo"
|
||||
(lib.lists.elemAt narinfoNameChars 0)
|
||||
((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1))
|
||||
narinfoName
|
||||
"xl.meta"
|
||||
];
|
||||
in
|
||||
''
|
||||
minio.start()
|
||||
|
||||
minio.wait_for_unit("minio.service")
|
||||
|
||||
minio.wait_until_succeeds("init-minio.sh")
|
||||
|
||||
start_all()
|
||||
|
||||
harmonia.wait_for_unit("harmonia.service")
|
||||
|
||||
ncps.wait_for_unit("ncps.service")
|
||||
|
||||
client.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2")
|
||||
|
||||
client.succeed("cat /etc/nix/nix.conf >&2")
|
||||
client.succeed("nix-store --realise ${pkgs.emptyFile}")
|
||||
|
||||
minio.succeed("cat ${narinfoPath} >&2")
|
||||
'';
|
||||
}
|
||||
Reference in New Issue
Block a user