nixos/tests/litestream: fix for v0.5.x

Litestream 0.5.x writes to the database (_litestream_seq table),
requiring group-write permissions on the data directory. Update the
test to use tmpfiles for permissions, pre-create the DB in WAL mode,
enable the IPC socket for deterministic sync, and verify backup
content via restore + dump.

nixos/tests/litestream.nix

@@ -1,4 +1,8 @@
-{ pkgs, ... }:
+{ pkgs, lib, ... }:
+let
+  dbPath = "/var/lib/grafana/data/grafana.db";
+  socketPath = "/run/litestream/litestream.sock";
+in
 {
   name = "litestream";
   meta = with pkgs.lib.maintainers; {
@@ -11,38 +15,45 @@
       services.litestream = {
         enable = true;
         settings = {
+          socket = {
+            enabled = true;
+            path = "${socketPath}";
+          };
           dbs = [
             {
-              path = "/var/lib/grafana/data/grafana.db";
+              path = dbPath;
               replicas = [
                 {
                   url = "sftp://foo:bar@127.0.0.1:22/home/foo/grafana";
+                  auto-recover = true;
                 }
               ];
             }
           ];
         };
       };
-      systemd.services.grafana.serviceConfig.ExecStartPost =
-        "+"
-        + pkgs.writeShellScript "grant-grafana-permissions" ''
-          timeout=10
-
-          while [ ! -f /var/lib/grafana/data/grafana.db ];
-          do
-            if [ "$timeout" == 0 ]; then
-              echo "ERROR: Timeout while waiting for /var/lib/grafana/data/grafana.db."
-              exit 1
-            fi
-
-            sleep 1
-
-            ((timeout--))
-          done
-
-          find /var/lib/grafana -type d -exec chmod -v 775 {} \;
-          find /var/lib/grafana -type f -exec chmod -v 660 {} \;
-        '';
+      # Litestream 0.5.x writes to the database (_litestream_seq table),
+      # so grafana's data directory must be group-writable.
+      systemd.tmpfiles.rules = [
+        "d /var/lib/grafana 0750 grafana grafana -"
+        "d /var/lib/grafana/data 2770 grafana grafana -"
+      ];
+      systemd.services.grafana.serviceConfig = {
+        ExecStartPre = lib.mkAfter "+${pkgs.sqlite}/bin/sqlite3 ${dbPath} 'PRAGMA journal_mode=WAL;'";
+        UMask = lib.mkForce "0007";
+      };
+      systemd.services.litestream = {
+        after = [
+          "grafana.service"
+          "sshd.service"
+        ];
+        requires = [ "grafana.service" ];
+        wants = [ "sshd.service" ];
+        serviceConfig = {
+          RuntimeDirectory = "litestream";
+          ExecStartPre = "+/bin/sh -c 'chmod g+rw ${dbPath}*'";
+        };
+      };
       services.openssh = {
         enable = true;
         allowSFTP = true;
@@ -69,7 +80,7 @@
 
           database = {
             type = "sqlite3";
-            path = "/var/lib/grafana/data/grafana.db";
+            path = dbPath;
             wal = true;
           };
         };
@@ -79,35 +90,44 @@
         password = "bar";
       };
       users.users.litestream.extraGroups = [ "grafana" ];
+      environment.systemPackages = [ pkgs.sqlite ];
     };
 
   testScript = ''
     start_all()
-    machine.wait_until_succeeds("test -d /home/foo/grafana")
     machine.wait_for_open_port(3000)
-    machine.succeed("""
-        curl -sSfN -X PUT -H "Content-Type: application/json" -d '{
-          "oldPassword": "admin",
-          "newPassword": "newpass",
-          "confirmNew": "newpass"
-        }' http://admin:admin@127.0.0.1:3000/api/user/password
-    """)
-    # https://litestream.io/guides/systemd/#simulating-a-disaster
-    machine.systemctl("stop litestream.service")
-    machine.succeed(
-        "rm -f /var/lib/grafana/data/grafana.db "
-        "/var/lib/grafana/data/grafana.db-shm "
-        "/var/lib/grafana/data/grafana.db-wal"
-    )
-    machine.succeed(
-        "litestream restore /var/lib/grafana/data/grafana.db "
-        "&& chown grafana:grafana /var/lib/grafana/data/grafana.db "
-        "&& chmod 660 /var/lib/grafana/data/grafana.db"
-    )
-    machine.systemctl("restart grafana.service")
-    machine.wait_for_open_port(3000)
-    machine.succeed(
-        "curl -sSfN -u admin:newpass http://127.0.0.1:3000/api/org/users | grep admin\@localhost"
-    )
+    with subtest("Verify litestream replicates changes"):
+        machine.succeed("""
+            curl -sSfN -X PUT --json '{
+              "name": "LitestreamTest",
+              "login": "admin",
+              "email": "admin@localhost"
+            }' http://admin:admin@127.0.0.1:3000/api/user
+        """)
+        machine.succeed("litestream sync -wait -socket ${socketPath} ${dbPath}")
+        machine.succeed(
+            "litestream restore -o /tmp/restored.db ${dbPath} && "
+            "sqlite3 /tmp/restored.db '.dump' | grep -q LitestreamTest"
+        )
+
+    with subtest("Simulate disaster recovery"):
+        # https://litestream.io/guides/systemd/#simulating-a-disaster
+        machine.systemctl("stop litestream.service grafana.service")
+        machine.succeed(
+            "rm -f ${dbPath} "
+            "${dbPath}-shm "
+            "${dbPath}-wal"
+        )
+        machine.succeed(
+            "litestream restore ${dbPath} "
+            "&& chown grafana:grafana ${dbPath} "
+            "&& chmod 660 ${dbPath}"
+        )
+        machine.systemctl("restart grafana.service litestream.service")
+        machine.wait_for_open_port(3000)
+        machine.wait_for_unit("litestream.service")
+        machine.succeed(
+            "curl -sSfN -u admin:admin http://127.0.0.1:3000/api/user | grep LitestreamTest"
+        )
   '';
 }