diff --git a/nixos/tests/litestream.nix b/nixos/tests/litestream.nix index b610c334ec01..6161742b539e 100644 --- a/nixos/tests/litestream.nix +++ b/nixos/tests/litestream.nix @@ -1,4 +1,8 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: +let + dbPath = "/var/lib/grafana/data/grafana.db"; + socketPath = "/run/litestream/litestream.sock"; +in { name = "litestream"; meta = with pkgs.lib.maintainers; { @@ -11,38 +15,45 @@ services.litestream = { enable = true; settings = { + socket = { + enabled = true; + path = "${socketPath}"; + }; dbs = [ { - path = "/var/lib/grafana/data/grafana.db"; + path = dbPath; replicas = [ { url = "sftp://foo:bar@127.0.0.1:22/home/foo/grafana"; + auto-recover = true; } ]; } ]; }; }; - systemd.services.grafana.serviceConfig.ExecStartPost = - "+" - + pkgs.writeShellScript "grant-grafana-permissions" '' - timeout=10 - - while [ ! -f /var/lib/grafana/data/grafana.db ]; - do - if [ "$timeout" == 0 ]; then - echo "ERROR: Timeout while waiting for /var/lib/grafana/data/grafana.db." - exit 1 - fi - - sleep 1 - - ((timeout--)) - done - - find /var/lib/grafana -type d -exec chmod -v 775 {} \; - find /var/lib/grafana -type f -exec chmod -v 660 {} \; - ''; + # Litestream 0.5.x writes to the database (_litestream_seq table), + # so grafana's data directory must be group-writable. + systemd.tmpfiles.rules = [ + "d /var/lib/grafana 0750 grafana grafana -" + "d /var/lib/grafana/data 2770 grafana grafana -" + ]; + systemd.services.grafana.serviceConfig = { + ExecStartPre = lib.mkAfter "+${pkgs.sqlite}/bin/sqlite3 ${dbPath} 'PRAGMA journal_mode=WAL;'"; + UMask = lib.mkForce "0007"; + }; + systemd.services.litestream = { + after = [ + "grafana.service" + "sshd.service" + ]; + requires = [ "grafana.service" ]; + wants = [ "sshd.service" ]; + serviceConfig = { + RuntimeDirectory = "litestream"; + ExecStartPre = "+/bin/sh -c 'chmod g+rw ${dbPath}*'"; + }; + }; services.openssh = { enable = true; allowSFTP = true; @@ -69,7 +80,7 @@ database = { type = "sqlite3"; - path = "/var/lib/grafana/data/grafana.db"; + path = dbPath; wal = true; }; }; @@ -79,35 +90,44 @@ password = "bar"; }; users.users.litestream.extraGroups = [ "grafana" ]; + environment.systemPackages = [ pkgs.sqlite ]; }; testScript = '' start_all() - machine.wait_until_succeeds("test -d /home/foo/grafana") machine.wait_for_open_port(3000) - machine.succeed(""" - curl -sSfN -X PUT -H "Content-Type: application/json" -d '{ - "oldPassword": "admin", - "newPassword": "newpass", - "confirmNew": "newpass" - }' http://admin:admin@127.0.0.1:3000/api/user/password - """) - # https://litestream.io/guides/systemd/#simulating-a-disaster - machine.systemctl("stop litestream.service") - machine.succeed( - "rm -f /var/lib/grafana/data/grafana.db " - "/var/lib/grafana/data/grafana.db-shm " - "/var/lib/grafana/data/grafana.db-wal" - ) - machine.succeed( - "litestream restore /var/lib/grafana/data/grafana.db " - "&& chown grafana:grafana /var/lib/grafana/data/grafana.db " - "&& chmod 660 /var/lib/grafana/data/grafana.db" - ) - machine.systemctl("restart grafana.service") - machine.wait_for_open_port(3000) - machine.succeed( - "curl -sSfN -u admin:newpass http://127.0.0.1:3000/api/org/users | grep admin\@localhost" - ) + with subtest("Verify litestream replicates changes"): + machine.succeed(""" + curl -sSfN -X PUT --json '{ + "name": "LitestreamTest", + "login": "admin", + "email": "admin@localhost" + }' http://admin:admin@127.0.0.1:3000/api/user + """) + machine.succeed("litestream sync -wait -socket ${socketPath} ${dbPath}") + machine.succeed( + "litestream restore -o /tmp/restored.db ${dbPath} && " + "sqlite3 /tmp/restored.db '.dump' | grep -q LitestreamTest" + ) + + with subtest("Simulate disaster recovery"): + # https://litestream.io/guides/systemd/#simulating-a-disaster + machine.systemctl("stop litestream.service grafana.service") + machine.succeed( + "rm -f ${dbPath} " + "${dbPath}-shm " + "${dbPath}-wal" + ) + machine.succeed( + "litestream restore ${dbPath} " + "&& chown grafana:grafana ${dbPath} " + "&& chmod 660 ${dbPath}" + ) + machine.systemctl("restart grafana.service litestream.service") + machine.wait_for_open_port(3000) + machine.wait_for_unit("litestream.service") + machine.succeed( + "curl -sSfN -u admin:admin http://127.0.0.1:3000/api/user | grep LitestreamTest" + ) ''; }