diff --git a/nixos/modules/services/networking/ncps.nix b/nixos/modules/services/networking/ncps.nix index 83161c051b5d..16737d63626d 100644 --- a/nixos/modules/services/networking/ncps.nix +++ b/nixos/modules/services/networking/ncps.nix @@ -17,6 +17,23 @@ let "panic" ]; + ncpsWrapper = pkgs.writeShellScript "ncps-wrapper" '' + ${lib.optionalString (cfg.cache.storage.s3 != null) '' + export CACHE_STORAGE_S3_ACCESS_KEY_ID="$(cat "$CREDENTIALS_DIRECTORY/s3AccessKeyId")" + export CACHE_STORAGE_S3_SECRET_ACCESS_KEY="$(cat "$CREDENTIALS_DIRECTORY/s3SecretAccessKey")" + ''} + + ${lib.optionalString (cfg.cache.redis != null && cfg.cache.redis.passwordFile != null) '' + export CACHE_REDIS_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/redisPassword")" + ''} + + ${lib.optionalString (cfg.cache.databaseURLFile != null) '' + export CACHE_DATABASE_URL="$(cat "$CREDENTIALS_DIRECTORY/databaseURL")" + ''} + + exec ${lib.getExe cfg.package} "$@" + ''; + globalFlags = lib.concatStringsSep " " ( [ "--log-level='${cfg.logLevel}'" ] ++ (lib.optionals cfg.openTelemetry.enable ( @@ -27,19 +44,45 @@ let cfg.openTelemetry.grpcURL != null ) "--otel-grpc-url='${cfg.openTelemetry.grpcURL}'") )) - ++ (lib.optionals cfg.prometheus.enable [ - "--prometheus-enabled" - ]) + ++ (lib.optional cfg.prometheus.enable "--prometheus-enabled") + ++ (lib.optional (!cfg.analytics.reporting.enable) "--analytics-reporting-enabled=false") ); serveFlags = lib.concatStringsSep " " ( [ "--cache-hostname='${cfg.cache.hostName}'" - "--cache-data-path='${cfg.cache.dataPath}'" - "--cache-database-url='${cfg.cache.databaseURL}'" + "--cache-lock-backend='${cfg.cache.lock.backend}'" "--cache-temp-path='${cfg.cache.tempPath}'" "--server-addr='${cfg.server.addr}'" ] + ++ (lib.optional (cfg.cache.databaseURL != null) "--cache-database-url='${cfg.cache.databaseURL}'") + ++ (lib.optionals (cfg.cache.redis != null) ( + [ + "--cache-redis-addrs='${builtins.concatStringsSep "," cfg.cache.redis.addresses}'" + "--cache-redis-db='${builtins.toString cfg.cache.redis.database}'" + "--cache-redis-pool-size='${builtins.toString cfg.cache.redis.poolSize}'" + ] + ++ (lib.optional ( + cfg.cache.redis.username != null + ) "--cache-redis-username='${cfg.cache.redis.username}'") + ++ (lib.optional ( + cfg.cache.redis.password != null + ) "--cache-redis-password='${cfg.cache.redis.password}'") + ++ (lib.optional cfg.cache.redis.useTLS "--cache-redis-use-tls") + )) + ++ (lib.optional ( + cfg.cache.storage.s3 == null + ) "--cache-storage-local='${cfg.cache.storage.local}'") + ++ (lib.optionals (cfg.cache.storage.s3 != null) ( + [ + "--cache-storage-s3-bucket='${cfg.cache.storage.s3.bucket}'" + "--cache-storage-s3-endpoint='${cfg.cache.storage.s3.endpoint}'" + ] + ++ (lib.optional cfg.cache.storage.s3.forcePathStyle "--cache-storage-s3-force-path-style") + ++ (lib.optional ( + cfg.cache.storage.s3.region != null + ) "--cache-storage-s3-region='${cfg.cache.storage.s3.region}'") + )) ++ (lib.optional cfg.cache.allowDeleteVerb "--cache-allow-delete-verb") ++ (lib.optional cfg.cache.allowPutVerb "--cache-allow-put-verb") ++ (lib.optional (cfg.cache.maxSize != null) "--cache-max-size='${cfg.cache.maxSize}'") @@ -49,24 +92,59 @@ let ]) ++ (lib.optional (cfg.cache.secretKeyPath != null) "--cache-secret-key-path='%d/secretKey'") ++ (lib.optional (!cfg.cache.signNarinfo) "--cache-sign-narinfo='false'") - ++ (lib.forEach cfg.upstream.caches (url: "--upstream-cache='${url}'")) - ++ (lib.forEach cfg.upstream.publicKeys (pk: "--upstream-public-key='${pk}'")) + ++ (lib.optional ( + cfg.cache.upstream.dialerTimeout != null + ) "--cache-upstream-dialer-timeout='${cfg.cache.upstream.dialerTimeout}'") + ++ (lib.optional ( + cfg.cache.upstream.responseHeaderTimeout != null + ) "--cache-upstream-response-header-timeout='${cfg.cache.upstream.responseHeaderTimeout}'") + ++ (lib.forEach cfg.cache.upstream.publicKeys (pk: "--cache-upstream-public-key='${pk}'")) + ++ (lib.forEach cfg.cache.upstream.urls (url: "--cache-upstream-url='${url}'")) ++ (lib.optional (cfg.netrcFile != null) "--netrc-file='${cfg.netrcFile}'") ); - isSqlite = lib.strings.hasPrefix "sqlite:" cfg.cache.databaseURL; + isSqlite = cfg.cache.databaseURL != null && lib.strings.hasPrefix "sqlite:" cfg.cache.databaseURL; - dbPath = lib.removePrefix "sqlite:" cfg.cache.databaseURL; - dbDir = dirOf dbPath; + dbPath = if isSqlite then lib.removePrefix "sqlite:" cfg.cache.databaseURL else null; + dbDir = if isSqlite then dirOf dbPath else null; in { + imports = [ + (lib.mkRenamedOptionModule + [ "services" "ncps" "cache" "dataPath" ] + [ "services" "ncps" "cache" "storage" "local" ] + ) + + (lib.mkRenamedOptionModule + [ "services" "ncps" "upstream" "caches" ] + [ "services" "ncps" "cache" "upstream" "urls" ] + ) + + (lib.mkRenamedOptionModule + [ "services" "ncps" "upstream" "publicKeys" ] + [ "services" "ncps" "cache" "upstream" "publicKeys" ] + ) + + (lib.mkRemovedOptionModule [ + "services" + "ncps" + "dbmatePackage" + ] "dbmate is now wrapped within ncps package, you need to override ncps to change dbmate package") + ]; + options = { services.ncps = { enable = lib.mkEnableOption "ncps: Nix binary cache proxy service implemented in Go"; - package = lib.mkPackageOption pkgs "ncps" { }; + analytics.reporting.enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Enable reporting anonymous usage statistics (DB type, Lock type, Total Size) to the project maintainers. + ''; + }; - dbmatePackage = lib.mkPackageOption pkgs "dbmate" { }; + package = lib.mkPackageOption pkgs "ncps" { }; openTelemetry = { enable = lib.mkEnableOption "Enable OpenTelemetry logs, metrics, and tracing"; @@ -113,23 +191,23 @@ in ''; }; - dataPath = lib.mkOption { - type = lib.types.str; - default = "/var/lib/ncps"; - description = '' - The local directory for storing configuration and cached store paths - ''; - }; - databaseURL = lib.mkOption { - type = lib.types.str; - default = "sqlite:${cfg.cache.dataPath}/db/db.sqlite"; + type = lib.types.nullOr lib.types.str; + default = "sqlite:${cfg.cache.storage.local}/db/db.sqlite"; defaultText = "sqlite:/var/lib/ncps/db/db.sqlite"; description = '' The URL of the database (currently only SQLite is supported) ''; }; + databaseURLFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + File containing the URL of the database. + ''; + }; + lru = { schedule = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -155,6 +233,17 @@ in }; }; + lock.backend = lib.mkOption { + type = lib.types.enum [ + "local" + "redis" + ]; + default = "local"; + description = '' + Lock backend to use: 'local' (single instance), 'redis' (distributed). + ''; + }; + maxSize = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -165,8 +254,81 @@ in ''; }; + redis = lib.mkOption { + type = lib.types.nullOr ( + lib.types.submodule { + options = { + addresses = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = '' + ["redis0:6379" "redis1:6379"] + ''; + description = '' + A list of host:port for the Redis servers that are part of a cluster. + To use a single Redis instance, just set this to its single address. + ''; + }; + + database = lib.mkOption { + type = lib.types.int; + default = 0; + description = '' + Redis database number (0-15) + ''; + }; + + username = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Redis username for authentication (for Redis ACL). + ''; + }; + + password = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Redis password for authentication (for Redis ACL). + ''; + }; + + passwordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + File containing the redis password for authentication (for Redis ACL). + ''; + }; + + poolSize = lib.mkOption { + type = lib.types.int; + default = 10; + description = '' + Redis connection pool size. + ''; + }; + + useTLS = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Use TLS for Redis connection. + ''; + }; + }; + } + ); + + default = null; + + description = '' + Configure Redis. + ''; + }; + secretKeyPath = lib.mkOption { - type = lib.types.nullOr lib.types.str; + type = lib.types.nullOr lib.types.path; default = null; description = '' The path to load the secretKey for signing narinfos. Leave this @@ -174,8 +336,77 @@ in ''; }; + storage = { + local = lib.mkOption { + type = lib.types.path; + default = "/var/lib/ncps"; + description = '' + The local directory for storing configuration and cached store + paths. This is ignored if services.ncps.cache.storage.s3 is not + null. + ''; + }; + + s3 = lib.mkOption { + type = lib.types.nullOr ( + lib.types.submodule { + options = { + bucket = lib.mkOption { + type = lib.types.str; + description = '' + The name of the S3 bucket. + ''; + }; + + endpoint = lib.mkOption { + type = lib.types.str; + description = '' + S3-compatible endpoint URL with scheme. + ''; + example = "https://s3.amazonaws.com"; + }; + + forcePathStyle = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Force path-style S3 addressing (bucket/key vs key.bucket). + ''; + }; + + region = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + The S3 region. + ''; + }; + + accessKeyIdPath = lib.mkOption { + type = lib.types.path; + description = '' + The path to a file containing only the access-key-id. + ''; + }; + + secretAccessKeyPath = lib.mkOption { + type = lib.types.path; + description = '' + The path to a file containing only the secret-access-key. + ''; + }; + }; + } + ); + default = null; + description = '' + Use S3 for storage instead of local storage. + ''; + }; + }; + tempPath = lib.mkOption { - type = lib.types.str; + type = lib.types.path; default = "/tmp"; description = '' The path to the temporary directory that is used by the cache to download NAR files @@ -190,6 +421,45 @@ in Whether to sign narInfo files or passthru as-is from upstream ''; }; + + upstream = { + dialerTimeout = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Timeout for establishing TCP connections to upstream caches (e.g., 3s, 5s, 10s). + ''; + }; + + responseHeaderTimeout = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "5s"; + description = '' + Timeout for waiting for upstream server's response headers. + ''; + }; + + publicKeys = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; + description = '' + A list of public keys of upstream caches in the format + `host[-[0-9]*]:public-key`. This flag is used to verify the + signatures of store paths downloaded from upstream caches. + ''; + }; + + urls = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ "https://cache.nixos.org" ]; + description = '' + A list of URLs of upstream binary caches. + ''; + }; + }; + }; server = { @@ -202,29 +472,8 @@ in }; }; - upstream = { - caches = lib.mkOption { - type = lib.types.listOf lib.types.str; - example = [ "https://cache.nixos.org" ]; - description = '' - A list of URLs of upstream binary caches. - ''; - }; - - publicKeys = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - example = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; - description = '' - A list of public keys of upstream caches in the format - `host[-[0-9]*]:public-key`. This flag is used to verify the - signatures of store paths downloaded from upstream caches. - ''; - }; - }; - netrcFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; + type = lib.types.nullOr lib.types.path; default = null; example = "/etc/nix/netrc"; description = '' @@ -237,10 +486,27 @@ in config = lib.mkIf cfg.enable { assertions = [ + { + assertion = lib.xor (cfg.cache.databaseURL != null) (cfg.cache.databaseURLFile != null); + message = "You must specify exactly one of config.ncps.cache.databaseURL or config.ncps.cache.databaseURLFile"; + } { assertion = cfg.cache.lru.schedule == null || cfg.cache.maxSize != null; message = "You must specify config.ncps.cache.lru.schedule when config.ncps.cache.maxSize is set"; } + { + assertion = + cfg.cache.redis == null || cfg.cache.redis.password == null || cfg.cache.redis.passwordFile == null; + message = "You cannot specify both config.ncps.cache.redis.password and config.ncps.cache.redis.passwordFile"; + } + { + assertion = cfg.cache.lock.backend == "redis" -> cfg.cache.redis != null; + message = "You must specify config.ncps.cache.redis when config.ncps.cache.lock.backend is set to 'redis'"; + } + { + assertion = cfg.cache.redis != null -> cfg.cache.lock.backend == "redis"; + message = "You must set config.ncps.cache.lock.backend to 'redis' when config.ncps.cache.redis is set"; + } ]; users.users.ncps = { @@ -249,34 +515,23 @@ in }; users.groups.ncps = { }; - systemd.services.ncps-create-directories = { - description = "Created required directories by ncps"; - serviceConfig = { - Type = "oneshot"; - UMask = "0066"; - }; - script = - (lib.optionalString (cfg.cache.dataPath != "/var/lib/ncps") '' - if ! test -d ${cfg.cache.dataPath}; then - mkdir -p ${cfg.cache.dataPath} - chown ncps:ncps ${cfg.cache.dataPath} - fi - '') - + (lib.optionalString isSqlite '' - if ! test -d ${dbDir}; then - mkdir -p ${dbDir} - chown ncps:ncps ${dbDir} - fi - '') - + (lib.optionalString (cfg.cache.tempPath != "/tmp") '' - if ! test -d ${cfg.cache.tempPath}; then - mkdir -p ${cfg.cache.tempPath} - chown ncps:ncps ${cfg.cache.tempPath} - fi - ''); - wantedBy = [ "ncps.service" ]; - before = [ "ncps.service" ]; - }; + systemd.tmpfiles.settings.ncps = + let + perms = { + group = "ncps"; + mode = "0700"; + user = "ncps"; + }; + in + lib.mkMerge [ + (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local != "/var/lib/ncps") { + "${cfg.cache.storage.local}".d = perms; + }) + + (lib.mkIf isSqlite { "${dbDir}".d = perms; }) + + (lib.mkIf (cfg.cache.tempPath != "/tmp") { "${cfg.cache.tempPath}".d = perms; }) + ]; systemd.services.ncps = { description = "ncps binary cache proxy service"; @@ -286,31 +541,61 @@ in wantedBy = [ "multi-user.target" ]; preStart = '' - ${lib.getExe cfg.dbmatePackage} --migrations-dir=${cfg.package}/share/ncps/db/migrations --url=${cfg.cache.databaseURL} up + ${lib.optionalString (cfg.cache.databaseURLFile != null) '' + export DATABASE_URL="$(cat "$CREDENTIALS_DIRECTORY/databaseURL")" + ''} + ${lib.optionalString (cfg.cache.databaseURL != null) '' + export DATABASE_URL="${cfg.cache.databaseURL}" + ''} + echo ${cfg.package}/bin/dbmate-ncps up + ${cfg.package}/bin/dbmate-ncps up ''; serviceConfig = lib.mkMerge [ { - ExecStart = "${lib.getExe cfg.package} ${globalFlags} serve ${serveFlags}"; + ExecStart = "${ncpsWrapper} ${globalFlags} serve ${serveFlags}"; User = "ncps"; Group = "ncps"; Restart = "on-failure"; RuntimeDirectory = "ncps"; } - # credentials + # credentials for cache.secretKeyPath (lib.mkIf (cfg.cache.secretKeyPath != null) { - LoadCredential = "secretKey:${cfg.cache.secretKeyPath}"; + LoadCredential = lib.singleton "secretKey:${cfg.cache.secretKeyPath}"; + }) + + # credentials for cache.storage.s3 accessKeyIdPath and secretAccessKeyPath + (lib.mkIf (cfg.cache.storage.s3 != null) { + LoadCredential = [ + "s3AccessKeyId:${cfg.cache.storage.s3.accessKeyIdPath}" + "s3SecretAccessKey:${cfg.cache.storage.s3.secretAccessKeyPath}" + ]; + }) + + # credentials for Redis + (lib.mkIf (cfg.cache.redis != null && cfg.cache.redis.passwordFile != null) { + LoadCredential = lib.singleton "redisPassword:${cfg.cache.redis.passwordFile}"; + }) + + (lib.mkIf (cfg.cache.databaseURLFile != null) { + LoadCredential = lib.singleton "databaseURL:${cfg.cache.databaseURLFile}"; }) # ensure permissions on required directories - (lib.mkIf (cfg.cache.dataPath != "/var/lib/ncps") { - ReadWritePaths = [ cfg.cache.dataPath ]; + (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local != "/var/lib/ncps") { + ReadWritePaths = [ cfg.cache.storage.local ]; }) - (lib.mkIf (cfg.cache.dataPath == "/var/lib/ncps") { + (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local == "/var/lib/ncps") { StateDirectory = "ncps"; StateDirectoryMode = "0700"; }) + (lib.mkIf (cfg.cache.storage.s3 != null && isSqlite && lib.strings.hasPrefix "/var/lib/ncps" dbDir) + { + StateDirectory = "ncps"; + StateDirectoryMode = "0700"; + } + ) (lib.mkIf (isSqlite && !lib.strings.hasPrefix "/var/lib/ncps" dbDir) { ReadWritePaths = [ dbDir ]; }) @@ -357,7 +642,8 @@ in ]; unitConfig.RequiresMountsFor = lib.concatStringsSep " " ( - [ "${cfg.cache.dataPath}" ] ++ lib.optional isSqlite dbDir + (lib.optional (cfg.cache.storage.s3 == null) "${cfg.cache.storage.local}") + ++ (lib.optional isSqlite dbDir) ); }; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 3f93c0827657..3c0547d4578c 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1029,10 +1029,15 @@ in nbd = runTest ./nbd.nix; ncdns = runTest ./ncdns.nix; ncps = runTest ./ncps.nix; - ncps-custom-cache-datapath = runTest { + ncps-custom-sqlite-directory = runTest { imports = [ ./ncps.nix ]; - defaults.services.ncps.cache.dataPath = "/path/to/ncps"; + defaults.services.ncps.cache.databaseURL = "sqlite:/path/to/ncps/db.sqlite"; }; + ncps-custom-storage-local = runTest { + imports = [ ./ncps.nix ]; + defaults.services.ncps.cache.storage.local = "/path/to/ncps"; + }; + ncps-ha = runTest ./ncps-ha.nix; ndppd = runTest ./ndppd.nix; nebula-lighthouse-service = runTest ./nebula-lighthouse-service.nix; nebula.connectivity = runTest ./nebula/connectivity.nix; diff --git a/nixos/tests/ncps-ha.nix b/nixos/tests/ncps-ha.nix new file mode 100644 index 000000000000..4bf5c6832ab8 --- /dev/null +++ b/nixos/tests/ncps-ha.nix @@ -0,0 +1,218 @@ +{ + lib, + pkgs, + ... +}: +let + # s3 creds + bucket = "ncps"; + region = "us-west-1"; + accessKey = builtins.toFile "minio-access-key" "easy-key"; + secretKey = builtins.toFile "minio-secret-key" "easy-secret"; + + # pg creds + postgresPassword = "easypwd"; + + # redis creds + redisPassword = "easypwd"; + + initMinio = pkgs.writeShellScriptBin "init-minio.sh" '' + set -euo pipefail + + mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin + mc mb local/${bucket} + mc admin user svcacct add --access-key "$(cat ${accessKey})" --secret-key "$(cat ${secretKey})" local minioadmin + ''; + + ncpsAttrs = hostname: { + services.ncps = { + enable = true; + + analytics.reporting.enable = false; + + cache = { + hostName = hostname; + + databaseURL = "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable"; + + lock.backend = "redis"; + + secretKeyPath = builtins.toString ( + pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA==" + ); + + redis = { + addresses = [ "redis:6379" ]; + passwordFile = builtins.toFile "redis-password" redisPassword; + }; + + storage.s3 = { + inherit bucket region; + + endpoint = "http://minio:9000"; + + accessKeyIdPath = accessKey; + secretAccessKeyPath = secretKey; + }; + + upstream = { + urls = [ "http://harmonia:5000" ]; + publicKeys = [ + "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo=" + ]; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ 8501 ]; + }; +in +{ + name = "ncps-storage-s3"; + meta = with lib.maintainers; { + maintainers = [ + aciceri + kalbasit + ]; + }; + + nodes = { + client0 = { + nix.settings = { + substituters = lib.mkForce [ "http://ncps0:8501" ]; + trusted-public-keys = lib.mkForce [ + "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg=" + ]; + }; + }; + + client1 = { + nix.settings = { + substituters = lib.mkForce [ "http://ncps1:8501" ]; + trusted-public-keys = lib.mkForce [ + "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg=" + ]; + }; + }; + + harmonia = { + services.harmonia = { + enable = true; + signKeyPaths = [ + (pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==") + ]; + settings.priority = 35; + }; + + networking.firewall.allowedTCPPorts = [ 5000 ]; + system.extraDependencies = [ pkgs.emptyFile ]; + }; + + minio = { + services.minio = { + inherit region; + + enable = true; + }; + + networking.firewall.allowedTCPPorts = [ 9000 ]; + environment.systemPackages = [ + pkgs.minio-client + initMinio + ]; + }; + + ncps0 = lib.mkMerge [ + (ncpsAttrs "ncps0") + { + services.ncps.cache.databaseURL = lib.mkForce null; + services.ncps.cache.databaseURLFile = builtins.toFile "db-url" "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable"; + } + ]; + ncps1 = ncpsAttrs "ncps1"; + + postgres = { + services.postgresql = { + enable = true; + enableTCPIP = true; + authentication = '' + host all all all scram-sha-256 + ''; + initialScript = pkgs.writeText "init-postgres.sql" '' + CREATE DATABASE "ncps" WITH ENCODING = 'UTF8'; + CREATE ROLE "ncps" WITH LOGIN PASSWORD '${ + builtins.replaceStrings [ "'" ] [ "''" ] postgresPassword + }'; + ALTER DATABASE "ncps" OWNER TO "ncps"; + ''; + }; + + networking.firewall.allowedTCPPorts = [ 5432 ]; + }; + + redis = { + services.redis.servers.ncps = { + enable = true; + openFirewall = true; + port = 6379; + requirePass = redisPassword; + bind = null; + }; + }; + }; + + testScript = + { nodes, ... }: + let + narinfoName = + (lib.strings.removePrefix "/nix/store/" ( + lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath + )) + + ".narinfo"; + + narinfoNameChars = lib.strings.stringToCharacters narinfoName; + + narinfoPath = lib.concatStringsSep "/" [ + (builtins.head nodes.minio.services.minio.dataDir) + bucket + "store/narinfo" + (lib.lists.elemAt narinfoNameChars 0) + ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1)) + narinfoName + "xl.meta" + ]; + in + '' + harmonia.start() + minio.start() + postgres.start() + redis.start() + + minio.wait_for_unit("minio.service") + + minio.wait_until_succeeds("init-minio.sh") + + postgres.wait_for_unit("postgresql.service") + redis.wait_for_unit("redis-ncps.service") + + redis.wait_until_succeeds("redis-cli -h redis -p 6379 -a '${redisPassword}' ping") + + start_all() + + harmonia.wait_for_unit("harmonia.service") + + ncps0.wait_for_unit("ncps.service") + ncps1.wait_for_unit("ncps.service") + + client0.wait_until_succeeds("curl -f http://ncps0:8501/ | grep '\"hostname\":\"${toString nodes.ncps0.services.ncps.cache.hostName}\"' >&2") + client1.wait_until_succeeds("curl -f http://ncps1:8501/ | grep '\"hostname\":\"${toString nodes.ncps1.services.ncps.cache.hostName}\"' >&2") + + client0.succeed("cat /etc/nix/nix.conf >&2") + client0.succeed("nix-store --realise ${pkgs.emptyFile}") + + client1.succeed("cat /etc/nix/nix.conf >&2") + client1.succeed("nix-store --realise ${pkgs.emptyFile}") + + minio.succeed("cat ${narinfoPath} >&2") + ''; +} diff --git a/nixos/tests/ncps.nix b/nixos/tests/ncps.nix index 76f23ccfe802..c2528e55b136 100644 --- a/nixos/tests/ncps.nix +++ b/nixos/tests/ncps.nix @@ -6,6 +6,12 @@ { name = "ncps"; + meta = with lib.maintainers; { + maintainers = [ + aciceri + kalbasit + ]; + }; nodes = { harmonia = { @@ -25,25 +31,27 @@ services.ncps = { enable = true; + analytics.reporting.enable = false; + cache = { hostName = "ncps"; secretKeyPath = toString ( pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA==" ); - }; - upstream = { - caches = [ "http://harmonia:5000" ]; - publicKeys = [ - "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo=" - ]; + upstream = { + urls = [ "http://harmonia:5000" ]; + publicKeys = [ + "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo=" + ]; + }; }; }; networking.firewall.allowedTCPPorts = [ 8501 ]; }; - client01 = { + client = { nix.settings = { substituters = lib.mkForce [ "http://ncps:8501" ]; trusted-public-keys = lib.mkForce [ @@ -65,7 +73,7 @@ narinfoNameChars = lib.strings.stringToCharacters narinfoName; narinfoPath = lib.concatStringsSep "/" [ - nodes.ncps.services.ncps.cache.dataPath + nodes.ncps.services.ncps.cache.storage.local "store/narinfo" (lib.lists.elemAt narinfoNameChars 0) ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1)) @@ -79,10 +87,10 @@ ncps.wait_for_unit("ncps.service") - client01.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2") + client.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2") - client01.succeed("cat /etc/nix/nix.conf >&2") - client01.succeed("nix-store --realise ${pkgs.emptyFile}") + client.succeed("cat /etc/nix/nix.conf >&2") + client.succeed("nix-store --realise ${pkgs.emptyFile}") ncps.succeed("cat ${narinfoPath} >&2") '';