nixos/ncps: Add support for S3, PostgreSQL and Redis (#475423)

nixos/modules/services/networking/ncps.nix

@@ -17,6 +17,23 @@ let
     "panic"
   ];
 
+  ncpsWrapper = pkgs.writeShellScript "ncps-wrapper" ''
+    ${lib.optionalString (cfg.cache.storage.s3 != null) ''
+      export CACHE_STORAGE_S3_ACCESS_KEY_ID="$(cat "$CREDENTIALS_DIRECTORY/s3AccessKeyId")"
+      export CACHE_STORAGE_S3_SECRET_ACCESS_KEY="$(cat "$CREDENTIALS_DIRECTORY/s3SecretAccessKey")"
+    ''}
+
+    ${lib.optionalString (cfg.cache.redis != null && cfg.cache.redis.passwordFile != null) ''
+      export CACHE_REDIS_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/redisPassword")"
+    ''}
+
+    ${lib.optionalString (cfg.cache.databaseURLFile != null) ''
+      export CACHE_DATABASE_URL="$(cat "$CREDENTIALS_DIRECTORY/databaseURL")"
+    ''}
+
+    exec ${lib.getExe cfg.package} "$@"
+  '';
+
   globalFlags = lib.concatStringsSep " " (
     [ "--log-level='${cfg.logLevel}'" ]
     ++ (lib.optionals cfg.openTelemetry.enable (
@@ -27,19 +44,45 @@ let
         cfg.openTelemetry.grpcURL != null
       ) "--otel-grpc-url='${cfg.openTelemetry.grpcURL}'")
     ))
-    ++ (lib.optionals cfg.prometheus.enable [
-      "--prometheus-enabled"
-    ])
+    ++ (lib.optional cfg.prometheus.enable "--prometheus-enabled")
+    ++ (lib.optional (!cfg.analytics.reporting.enable) "--analytics-reporting-enabled=false")
   );
 
   serveFlags = lib.concatStringsSep " " (
     [
       "--cache-hostname='${cfg.cache.hostName}'"
-      "--cache-data-path='${cfg.cache.dataPath}'"
-      "--cache-database-url='${cfg.cache.databaseURL}'"
+      "--cache-lock-backend='${cfg.cache.lock.backend}'"
       "--cache-temp-path='${cfg.cache.tempPath}'"
       "--server-addr='${cfg.server.addr}'"
     ]
+    ++ (lib.optional (cfg.cache.databaseURL != null) "--cache-database-url='${cfg.cache.databaseURL}'")
+    ++ (lib.optionals (cfg.cache.redis != null) (
+      [
+        "--cache-redis-addrs='${builtins.concatStringsSep "," cfg.cache.redis.addresses}'"
+        "--cache-redis-db='${builtins.toString cfg.cache.redis.database}'"
+        "--cache-redis-pool-size='${builtins.toString cfg.cache.redis.poolSize}'"
+      ]
+      ++ (lib.optional (
+        cfg.cache.redis.username != null
+      ) "--cache-redis-username='${cfg.cache.redis.username}'")
+      ++ (lib.optional (
+        cfg.cache.redis.password != null
+      ) "--cache-redis-password='${cfg.cache.redis.password}'")
+      ++ (lib.optional cfg.cache.redis.useTLS "--cache-redis-use-tls")
+    ))
+    ++ (lib.optional (
+      cfg.cache.storage.s3 == null
+    ) "--cache-storage-local='${cfg.cache.storage.local}'")
+    ++ (lib.optionals (cfg.cache.storage.s3 != null) (
+      [
+        "--cache-storage-s3-bucket='${cfg.cache.storage.s3.bucket}'"
+        "--cache-storage-s3-endpoint='${cfg.cache.storage.s3.endpoint}'"
+      ]
+      ++ (lib.optional cfg.cache.storage.s3.forcePathStyle "--cache-storage-s3-force-path-style")
+      ++ (lib.optional (
+        cfg.cache.storage.s3.region != null
+      ) "--cache-storage-s3-region='${cfg.cache.storage.s3.region}'")
+    ))
     ++ (lib.optional cfg.cache.allowDeleteVerb "--cache-allow-delete-verb")
     ++ (lib.optional cfg.cache.allowPutVerb "--cache-allow-put-verb")
     ++ (lib.optional (cfg.cache.maxSize != null) "--cache-max-size='${cfg.cache.maxSize}'")
@@ -49,24 +92,59 @@ let
     ])
     ++ (lib.optional (cfg.cache.secretKeyPath != null) "--cache-secret-key-path='%d/secretKey'")
     ++ (lib.optional (!cfg.cache.signNarinfo) "--cache-sign-narinfo='false'")
-    ++ (lib.forEach cfg.upstream.caches (url: "--upstream-cache='${url}'"))
-    ++ (lib.forEach cfg.upstream.publicKeys (pk: "--upstream-public-key='${pk}'"))
+    ++ (lib.optional (
+      cfg.cache.upstream.dialerTimeout != null
+    ) "--cache-upstream-dialer-timeout='${cfg.cache.upstream.dialerTimeout}'")
+    ++ (lib.optional (
+      cfg.cache.upstream.responseHeaderTimeout != null
+    ) "--cache-upstream-response-header-timeout='${cfg.cache.upstream.responseHeaderTimeout}'")
+    ++ (lib.forEach cfg.cache.upstream.publicKeys (pk: "--cache-upstream-public-key='${pk}'"))
+    ++ (lib.forEach cfg.cache.upstream.urls (url: "--cache-upstream-url='${url}'"))
     ++ (lib.optional (cfg.netrcFile != null) "--netrc-file='${cfg.netrcFile}'")
   );
 
-  isSqlite = lib.strings.hasPrefix "sqlite:" cfg.cache.databaseURL;
+  isSqlite = cfg.cache.databaseURL != null && lib.strings.hasPrefix "sqlite:" cfg.cache.databaseURL;
 
-  dbPath = lib.removePrefix "sqlite:" cfg.cache.databaseURL;
-  dbDir = dirOf dbPath;
+  dbPath = if isSqlite then lib.removePrefix "sqlite:" cfg.cache.databaseURL else null;
+  dbDir = if isSqlite then dirOf dbPath else null;
 in
 {
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "ncps" "cache" "dataPath" ]
+      [ "services" "ncps" "cache" "storage" "local" ]
+    )
+
+    (lib.mkRenamedOptionModule
+      [ "services" "ncps" "upstream" "caches" ]
+      [ "services" "ncps" "cache" "upstream" "urls" ]
+    )
+
+    (lib.mkRenamedOptionModule
+      [ "services" "ncps" "upstream" "publicKeys" ]
+      [ "services" "ncps" "cache" "upstream" "publicKeys" ]
+    )
+
+    (lib.mkRemovedOptionModule [
+      "services"
+      "ncps"
+      "dbmatePackage"
+    ] "dbmate is now wrapped within ncps package, you need to override ncps to change dbmate package")
+  ];
+
   options = {
     services.ncps = {
       enable = lib.mkEnableOption "ncps: Nix binary cache proxy service implemented in Go";
 
-      package = lib.mkPackageOption pkgs "ncps" { };
+      analytics.reporting.enable = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Enable reporting anonymous usage statistics (DB type, Lock type, Total Size) to the project maintainers.
+        '';
+      };
 
-      dbmatePackage = lib.mkPackageOption pkgs "dbmate" { };
+      package = lib.mkPackageOption pkgs "ncps" { };
 
       openTelemetry = {
         enable = lib.mkEnableOption "Enable OpenTelemetry logs, metrics, and tracing";
@@ -113,23 +191,23 @@ in
           '';
         };
 
-        dataPath = lib.mkOption {
-          type = lib.types.str;
-          default = "/var/lib/ncps";
-          description = ''
-            The local directory for storing configuration and cached store paths
-          '';
-        };
-
         databaseURL = lib.mkOption {
-          type = lib.types.str;
-          default = "sqlite:${cfg.cache.dataPath}/db/db.sqlite";
+          type = lib.types.nullOr lib.types.str;
+          default = "sqlite:${cfg.cache.storage.local}/db/db.sqlite";
           defaultText = "sqlite:/var/lib/ncps/db/db.sqlite";
           description = ''
             The URL of the database (currently only SQLite is supported)
           '';
         };
 
+        databaseURLFile = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          description = ''
+            File containing the URL of the database.
+          '';
+        };
+
         lru = {
           schedule = lib.mkOption {
             type = lib.types.nullOr lib.types.str;
@@ -155,6 +233,17 @@ in
           };
         };
 
+        lock.backend = lib.mkOption {
+          type = lib.types.enum [
+            "local"
+            "redis"
+          ];
+          default = "local";
+          description = ''
+            Lock backend to use: 'local' (single instance), 'redis' (distributed).
+          '';
+        };
+
         maxSize = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
           default = null;
@@ -165,8 +254,81 @@ in
           '';
         };
 
+        redis = lib.mkOption {
+          type = lib.types.nullOr (
+            lib.types.submodule {
+              options = {
+                addresses = lib.mkOption {
+                  type = lib.types.listOf lib.types.str;
+                  example = ''
+                    ["redis0:6379" "redis1:6379"]
+                  '';
+                  description = ''
+                    A list of host:port for the Redis servers that are part of a cluster.
+                    To use a single Redis instance, just set this to its single address.
+                  '';
+                };
+
+                database = lib.mkOption {
+                  type = lib.types.int;
+                  default = 0;
+                  description = ''
+                    Redis database number (0-15)
+                  '';
+                };
+
+                username = lib.mkOption {
+                  type = lib.types.nullOr lib.types.str;
+                  default = null;
+                  description = ''
+                    Redis username for authentication (for Redis ACL).
+                  '';
+                };
+
+                password = lib.mkOption {
+                  type = lib.types.nullOr lib.types.str;
+                  default = null;
+                  description = ''
+                    Redis password for authentication (for Redis ACL).
+                  '';
+                };
+
+                passwordFile = lib.mkOption {
+                  type = lib.types.nullOr lib.types.path;
+                  default = null;
+                  description = ''
+                    File containing the redis password for authentication (for Redis ACL).
+                  '';
+                };
+
+                poolSize = lib.mkOption {
+                  type = lib.types.int;
+                  default = 10;
+                  description = ''
+                    Redis connection pool size.
+                  '';
+                };
+
+                useTLS = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Use TLS for Redis connection.
+                  '';
+                };
+              };
+            }
+          );
+
+          default = null;
+
+          description = ''
+            Configure Redis.
+          '';
+        };
+
         secretKeyPath = lib.mkOption {
-          type = lib.types.nullOr lib.types.str;
+          type = lib.types.nullOr lib.types.path;
           default = null;
           description = ''
             The path to load the secretKey for signing narinfos. Leave this
@@ -174,8 +336,77 @@ in
           '';
         };
 
+        storage = {
+          local = lib.mkOption {
+            type = lib.types.path;
+            default = "/var/lib/ncps";
+            description = ''
+              The local directory for storing configuration and cached store
+              paths. This is ignored if services.ncps.cache.storage.s3 is not
+              null.
+            '';
+          };
+
+          s3 = lib.mkOption {
+            type = lib.types.nullOr (
+              lib.types.submodule {
+                options = {
+                  bucket = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      The name of the S3 bucket.
+                    '';
+                  };
+
+                  endpoint = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      S3-compatible endpoint URL with scheme.
+                    '';
+                    example = "https://s3.amazonaws.com";
+                  };
+
+                  forcePathStyle = lib.mkOption {
+                    type = lib.types.bool;
+                    default = false;
+                    description = ''
+                      Force path-style S3 addressing (bucket/key vs key.bucket).
+                    '';
+                  };
+
+                  region = lib.mkOption {
+                    type = lib.types.nullOr lib.types.str;
+                    default = null;
+                    description = ''
+                      The S3 region.
+                    '';
+                  };
+
+                  accessKeyIdPath = lib.mkOption {
+                    type = lib.types.path;
+                    description = ''
+                      The path to a file containing only the access-key-id.
+                    '';
+                  };
+
+                  secretAccessKeyPath = lib.mkOption {
+                    type = lib.types.path;
+                    description = ''
+                      The path to a file containing only the secret-access-key.
+                    '';
+                  };
+                };
+              }
+            );
+            default = null;
+            description = ''
+              Use S3 for storage instead of local storage.
+            '';
+          };
+        };
+
         tempPath = lib.mkOption {
-          type = lib.types.str;
+          type = lib.types.path;
           default = "/tmp";
           description = ''
             The path to the temporary directory that is used by the cache to download NAR files
@@ -190,6 +421,45 @@ in
             Whether to sign narInfo files or passthru as-is from upstream
           '';
         };
+
+        upstream = {
+          dialerTimeout = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            default = null;
+            description = ''
+              Timeout for establishing TCP connections to upstream caches (e.g., 3s, 5s, 10s).
+            '';
+          };
+
+          responseHeaderTimeout = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            default = null;
+            example = "5s";
+            description = ''
+              Timeout for waiting for upstream server's response headers.
+            '';
+          };
+
+          publicKeys = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
+            description = ''
+              A list of public keys of upstream caches in the format
+              `host[-[0-9]*]:public-key`. This flag is used to verify the
+              signatures of store paths downloaded from upstream caches.
+            '';
+          };
+
+          urls = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            example = [ "https://cache.nixos.org" ];
+            description = ''
+              A list of URLs of upstream binary caches.
+            '';
+          };
+        };
+
       };
 
       server = {
@@ -202,29 +472,8 @@ in
         };
       };
 
-      upstream = {
-        caches = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          example = [ "https://cache.nixos.org" ];
-          description = ''
-            A list of URLs of upstream binary caches.
-          '';
-        };
-
-        publicKeys = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
-          default = [ ];
-          example = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
-          description = ''
-            A list of public keys of upstream caches in the format
-            `host[-[0-9]*]:public-key`. This flag is used to verify the
-            signatures of store paths downloaded from upstream caches.
-          '';
-        };
-      };
-
       netrcFile = lib.mkOption {
-        type = lib.types.nullOr lib.types.str;
+        type = lib.types.nullOr lib.types.path;
         default = null;
         example = "/etc/nix/netrc";
         description = ''
@@ -237,10 +486,27 @@ in
 
   config = lib.mkIf cfg.enable {
     assertions = [
+      {
+        assertion = lib.xor (cfg.cache.databaseURL != null) (cfg.cache.databaseURLFile != null);
+        message = "You must specify exactly one of config.ncps.cache.databaseURL or config.ncps.cache.databaseURLFile";
+      }
       {
         assertion = cfg.cache.lru.schedule == null || cfg.cache.maxSize != null;
         message = "You must specify config.ncps.cache.lru.schedule when config.ncps.cache.maxSize is set";
       }
+      {
+        assertion =
+          cfg.cache.redis == null || cfg.cache.redis.password == null || cfg.cache.redis.passwordFile == null;
+        message = "You cannot specify both config.ncps.cache.redis.password and config.ncps.cache.redis.passwordFile";
+      }
+      {
+        assertion = cfg.cache.lock.backend == "redis" -> cfg.cache.redis != null;
+        message = "You must specify config.ncps.cache.redis when config.ncps.cache.lock.backend is set to 'redis'";
+      }
+      {
+        assertion = cfg.cache.redis != null -> cfg.cache.lock.backend == "redis";
+        message = "You must set config.ncps.cache.lock.backend to 'redis' when config.ncps.cache.redis is set";
+      }
     ];
 
     users.users.ncps = {
@@ -249,34 +515,23 @@ in
     };
     users.groups.ncps = { };
 
-    systemd.services.ncps-create-directories = {
-      description = "Created required directories by ncps";
-      serviceConfig = {
-        Type = "oneshot";
-        UMask = "0066";
-      };
-      script =
-        (lib.optionalString (cfg.cache.dataPath != "/var/lib/ncps") ''
-          if ! test -d ${cfg.cache.dataPath}; then
-            mkdir -p ${cfg.cache.dataPath}
-            chown ncps:ncps ${cfg.cache.dataPath}
-          fi
-        '')
-        + (lib.optionalString isSqlite ''
-          if ! test -d ${dbDir}; then
-            mkdir -p ${dbDir}
-            chown ncps:ncps ${dbDir}
-          fi
-        '')
-        + (lib.optionalString (cfg.cache.tempPath != "/tmp") ''
-          if ! test -d ${cfg.cache.tempPath}; then
-            mkdir -p ${cfg.cache.tempPath}
-            chown ncps:ncps ${cfg.cache.tempPath}
-          fi
-        '');
-      wantedBy = [ "ncps.service" ];
-      before = [ "ncps.service" ];
-    };
+    systemd.tmpfiles.settings.ncps =
+      let
+        perms = {
+          group = "ncps";
+          mode = "0700";
+          user = "ncps";
+        };
+      in
+      lib.mkMerge [
+        (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local != "/var/lib/ncps") {
+          "${cfg.cache.storage.local}".d = perms;
+        })
+
+        (lib.mkIf isSqlite { "${dbDir}".d = perms; })
+
+        (lib.mkIf (cfg.cache.tempPath != "/tmp") { "${cfg.cache.tempPath}".d = perms; })
+      ];
 
     systemd.services.ncps = {
       description = "ncps binary cache proxy service";
@@ -286,31 +541,61 @@ in
       wantedBy = [ "multi-user.target" ];
 
       preStart = ''
-        ${lib.getExe cfg.dbmatePackage} --migrations-dir=${cfg.package}/share/ncps/db/migrations --url=${cfg.cache.databaseURL} up
+        ${lib.optionalString (cfg.cache.databaseURLFile != null) ''
+          export DATABASE_URL="$(cat "$CREDENTIALS_DIRECTORY/databaseURL")"
+        ''}
+        ${lib.optionalString (cfg.cache.databaseURL != null) ''
+          export DATABASE_URL="${cfg.cache.databaseURL}"
+        ''}
+        echo ${cfg.package}/bin/dbmate-ncps up
+        ${cfg.package}/bin/dbmate-ncps up
       '';
 
       serviceConfig = lib.mkMerge [
         {
-          ExecStart = "${lib.getExe cfg.package} ${globalFlags} serve ${serveFlags}";
+          ExecStart = "${ncpsWrapper} ${globalFlags} serve ${serveFlags}";
           User = "ncps";
           Group = "ncps";
           Restart = "on-failure";
           RuntimeDirectory = "ncps";
         }
 
-        # credentials
+        # credentials for cache.secretKeyPath
         (lib.mkIf (cfg.cache.secretKeyPath != null) {
-          LoadCredential = "secretKey:${cfg.cache.secretKeyPath}";
+          LoadCredential = lib.singleton "secretKey:${cfg.cache.secretKeyPath}";
+        })
+
+        # credentials for cache.storage.s3 accessKeyIdPath and secretAccessKeyPath
+        (lib.mkIf (cfg.cache.storage.s3 != null) {
+          LoadCredential = [
+            "s3AccessKeyId:${cfg.cache.storage.s3.accessKeyIdPath}"
+            "s3SecretAccessKey:${cfg.cache.storage.s3.secretAccessKeyPath}"
+          ];
+        })
+
+        # credentials for Redis
+        (lib.mkIf (cfg.cache.redis != null && cfg.cache.redis.passwordFile != null) {
+          LoadCredential = lib.singleton "redisPassword:${cfg.cache.redis.passwordFile}";
+        })
+
+        (lib.mkIf (cfg.cache.databaseURLFile != null) {
+          LoadCredential = lib.singleton "databaseURL:${cfg.cache.databaseURLFile}";
         })
 
         # ensure permissions on required directories
-        (lib.mkIf (cfg.cache.dataPath != "/var/lib/ncps") {
-          ReadWritePaths = [ cfg.cache.dataPath ];
+        (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local != "/var/lib/ncps") {
+          ReadWritePaths = [ cfg.cache.storage.local ];
         })
-        (lib.mkIf (cfg.cache.dataPath == "/var/lib/ncps") {
+        (lib.mkIf (cfg.cache.storage.s3 == null && cfg.cache.storage.local == "/var/lib/ncps") {
           StateDirectory = "ncps";
           StateDirectoryMode = "0700";
         })
+        (lib.mkIf (cfg.cache.storage.s3 != null && isSqlite && lib.strings.hasPrefix "/var/lib/ncps" dbDir)
+          {
+            StateDirectory = "ncps";
+            StateDirectoryMode = "0700";
+          }
+        )
         (lib.mkIf (isSqlite && !lib.strings.hasPrefix "/var/lib/ncps" dbDir) {
           ReadWritePaths = [ dbDir ];
         })
@@ -357,7 +642,8 @@ in
       ];
 
       unitConfig.RequiresMountsFor = lib.concatStringsSep " " (
-        [ "${cfg.cache.dataPath}" ] ++ lib.optional isSqlite dbDir
+        (lib.optional (cfg.cache.storage.s3 == null) "${cfg.cache.storage.local}")
+        ++ (lib.optional isSqlite dbDir)
       );
     };
   };

nixos/tests/all-tests.nix

@@ -1029,10 +1029,15 @@ in
   nbd = runTest ./nbd.nix;
   ncdns = runTest ./ncdns.nix;
   ncps = runTest ./ncps.nix;
-  ncps-custom-cache-datapath = runTest {
+  ncps-custom-sqlite-directory = runTest {
     imports = [ ./ncps.nix ];
-    defaults.services.ncps.cache.dataPath = "/path/to/ncps";
+    defaults.services.ncps.cache.databaseURL = "sqlite:/path/to/ncps/db.sqlite";
   };
+  ncps-custom-storage-local = runTest {
+    imports = [ ./ncps.nix ];
+    defaults.services.ncps.cache.storage.local = "/path/to/ncps";
+  };
+  ncps-ha = runTest ./ncps-ha.nix;
   ndppd = runTest ./ndppd.nix;
   nebula-lighthouse-service = runTest ./nebula-lighthouse-service.nix;
   nebula.connectivity = runTest ./nebula/connectivity.nix;

nixos/tests/ncps-ha.nix

@@ -0,0 +1,218 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  # s3 creds
+  bucket = "ncps";
+  region = "us-west-1";
+  accessKey = builtins.toFile "minio-access-key" "easy-key";
+  secretKey = builtins.toFile "minio-secret-key" "easy-secret";
+
+  # pg creds
+  postgresPassword = "easypwd";
+
+  # redis creds
+  redisPassword = "easypwd";
+
+  initMinio = pkgs.writeShellScriptBin "init-minio.sh" ''
+    set -euo pipefail
+
+    mc alias set local "http://127.0.0.1:9000" minioadmin minioadmin
+    mc mb local/${bucket}
+    mc admin user svcacct add --access-key "$(cat ${accessKey})" --secret-key "$(cat ${secretKey})" local minioadmin
+  '';
+
+  ncpsAttrs = hostname: {
+    services.ncps = {
+      enable = true;
+
+      analytics.reporting.enable = false;
+
+      cache = {
+        hostName = hostname;
+
+        databaseURL = "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable";
+
+        lock.backend = "redis";
+
+        secretKeyPath = builtins.toString (
+          pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA=="
+        );
+
+        redis = {
+          addresses = [ "redis:6379" ];
+          passwordFile = builtins.toFile "redis-password" redisPassword;
+        };
+
+        storage.s3 = {
+          inherit bucket region;
+
+          endpoint = "http://minio:9000";
+
+          accessKeyIdPath = accessKey;
+          secretAccessKeyPath = secretKey;
+        };
+
+        upstream = {
+          urls = [ "http://harmonia:5000" ];
+          publicKeys = [
+            "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
+          ];
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 8501 ];
+  };
+in
+{
+  name = "ncps-storage-s3";
+  meta = with lib.maintainers; {
+    maintainers = [
+      aciceri
+      kalbasit
+    ];
+  };
+
+  nodes = {
+    client0 = {
+      nix.settings = {
+        substituters = lib.mkForce [ "http://ncps0:8501" ];
+        trusted-public-keys = lib.mkForce [
+          "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
+        ];
+      };
+    };
+
+    client1 = {
+      nix.settings = {
+        substituters = lib.mkForce [ "http://ncps1:8501" ];
+        trusted-public-keys = lib.mkForce [
+          "ncps:UtiE6C+3Tx0kgpP34vjyX/BKK6QZ/D1OzDYX72aCPJg="
+        ];
+      };
+    };
+
+    harmonia = {
+      services.harmonia = {
+        enable = true;
+        signKeyPaths = [
+          (pkgs.writeText "cache-key" "cache.example.com-1:9FhO0w+7HjZrhvmzT1VlAZw4OSAlFGTgC24Seg3tmPl4gZBdwZClzTTHr9cVzJpwsRSYLTu7hEAQe3ljy92CWg==")
+        ];
+        settings.priority = 35;
+      };
+
+      networking.firewall.allowedTCPPorts = [ 5000 ];
+      system.extraDependencies = [ pkgs.emptyFile ];
+    };
+
+    minio = {
+      services.minio = {
+        inherit region;
+
+        enable = true;
+      };
+
+      networking.firewall.allowedTCPPorts = [ 9000 ];
+      environment.systemPackages = [
+        pkgs.minio-client
+        initMinio
+      ];
+    };
+
+    ncps0 = lib.mkMerge [
+      (ncpsAttrs "ncps0")
+      {
+        services.ncps.cache.databaseURL = lib.mkForce null;
+        services.ncps.cache.databaseURLFile = builtins.toFile "db-url" "postgres://ncps:${lib.escapeURL postgresPassword}@postgres:5432/ncps?sslmode=disable";
+      }
+    ];
+    ncps1 = ncpsAttrs "ncps1";
+
+    postgres = {
+      services.postgresql = {
+        enable = true;
+        enableTCPIP = true;
+        authentication = ''
+          host all all all scram-sha-256
+        '';
+        initialScript = pkgs.writeText "init-postgres.sql" ''
+          CREATE DATABASE "ncps" WITH ENCODING = 'UTF8';
+          CREATE ROLE "ncps" WITH LOGIN PASSWORD '${
+            builtins.replaceStrings [ "'" ] [ "''" ] postgresPassword
+          }';
+          ALTER DATABASE "ncps" OWNER TO "ncps";
+        '';
+      };
+
+      networking.firewall.allowedTCPPorts = [ 5432 ];
+    };
+
+    redis = {
+      services.redis.servers.ncps = {
+        enable = true;
+        openFirewall = true;
+        port = 6379;
+        requirePass = redisPassword;
+        bind = null;
+      };
+    };
+  };
+
+  testScript =
+    { nodes, ... }:
+    let
+      narinfoName =
+        (lib.strings.removePrefix "/nix/store/" (
+          lib.strings.removeSuffix "-empty-file" pkgs.emptyFile.outPath
+        ))
+        + ".narinfo";
+
+      narinfoNameChars = lib.strings.stringToCharacters narinfoName;
+
+      narinfoPath = lib.concatStringsSep "/" [
+        (builtins.head nodes.minio.services.minio.dataDir)
+        bucket
+        "store/narinfo"
+        (lib.lists.elemAt narinfoNameChars 0)
+        ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1))
+        narinfoName
+        "xl.meta"
+      ];
+    in
+    ''
+      harmonia.start()
+      minio.start()
+      postgres.start()
+      redis.start()
+
+      minio.wait_for_unit("minio.service")
+
+      minio.wait_until_succeeds("init-minio.sh")
+
+      postgres.wait_for_unit("postgresql.service")
+      redis.wait_for_unit("redis-ncps.service")
+
+      redis.wait_until_succeeds("redis-cli -h redis -p 6379 -a '${redisPassword}' ping")
+
+      start_all()
+
+      harmonia.wait_for_unit("harmonia.service")
+
+      ncps0.wait_for_unit("ncps.service")
+      ncps1.wait_for_unit("ncps.service")
+
+      client0.wait_until_succeeds("curl -f http://ncps0:8501/ | grep '\"hostname\":\"${toString nodes.ncps0.services.ncps.cache.hostName}\"' >&2")
+      client1.wait_until_succeeds("curl -f http://ncps1:8501/ | grep '\"hostname\":\"${toString nodes.ncps1.services.ncps.cache.hostName}\"' >&2")
+
+      client0.succeed("cat /etc/nix/nix.conf >&2")
+      client0.succeed("nix-store --realise ${pkgs.emptyFile}")
+
+      client1.succeed("cat /etc/nix/nix.conf >&2")
+      client1.succeed("nix-store --realise ${pkgs.emptyFile}")
+
+      minio.succeed("cat ${narinfoPath} >&2")
+    '';
+}

nixos/tests/ncps.nix

@@ -6,6 +6,12 @@
 
 {
   name = "ncps";
+  meta = with lib.maintainers; {
+    maintainers = [
+      aciceri
+      kalbasit
+    ];
+  };
 
   nodes = {
     harmonia = {
@@ -25,25 +31,27 @@
       services.ncps = {
         enable = true;
 
+        analytics.reporting.enable = false;
+
         cache = {
           hostName = "ncps";
           secretKeyPath = toString (
             pkgs.writeText "ncps-cache-key" "ncps:dcrGsrku0KvltFhrR5lVIMqyloAdo0y8vYZOeIFUSLJS2IToL7dPHSSCk/fi+PJf8EorpBn8PU7MNhfvZoI8mA=="
           );
-        };
 
-        upstream = {
-          caches = [ "http://harmonia:5000" ];
-          publicKeys = [
-            "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
-          ];
+          upstream = {
+            urls = [ "http://harmonia:5000" ];
+            publicKeys = [
+              "cache.example.com-1:eIGQXcGQpc00x6/XFcyacLEUmC07u4RAEHt5Y8vdglo="
+            ];
+          };
         };
       };
 
       networking.firewall.allowedTCPPorts = [ 8501 ];
     };
 
-    client01 = {
+    client = {
       nix.settings = {
         substituters = lib.mkForce [ "http://ncps:8501" ];
         trusted-public-keys = lib.mkForce [
@@ -65,7 +73,7 @@
       narinfoNameChars = lib.strings.stringToCharacters narinfoName;
 
       narinfoPath = lib.concatStringsSep "/" [
-        nodes.ncps.services.ncps.cache.dataPath
+        nodes.ncps.services.ncps.cache.storage.local
         "store/narinfo"
         (lib.lists.elemAt narinfoNameChars 0)
         ((lib.lists.elemAt narinfoNameChars 0) + (lib.lists.elemAt narinfoNameChars 1))
@@ -79,10 +87,10 @@
 
       ncps.wait_for_unit("ncps.service")
 
-      client01.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2")
+      client.wait_until_succeeds("curl -f http://ncps:8501/ | grep '\"hostname\":\"${toString nodes.ncps.services.ncps.cache.hostName}\"' >&2")
 
-      client01.succeed("cat /etc/nix/nix.conf >&2")
-      client01.succeed("nix-store --realise ${pkgs.emptyFile}")
+      client.succeed("cat /etc/nix/nix.conf >&2")
+      client.succeed("nix-store --realise ${pkgs.emptyFile}")
 
       ncps.succeed("cat ${narinfoPath} >&2")
     '';