From c6994e90dcb463af57917780838cc4d0dc87a2b7 Mon Sep 17 00:00:00 2001 From: Matt McHenry Date: Sat, 21 Jul 2018 22:24:19 -0400 Subject: [PATCH 1/6] restic: add support for pruning --- nixos/modules/services/backup/restic.nix | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index 7e8e91e4b9c3..a0af74f66997 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -103,6 +103,23 @@ in Create the repository if it doesn't exist. ''; }; + + pruneOpts = mkOption { + type = types.listOf types.str; + default = []; + description = '' + A list of options (--keep-* et al.) for 'restic forget + --prune', to automatically prune old snapshots. The + 'forget' command is run *after* the 'backup' command, so + keep that in mind when constructing the --keep-* options. + ''; + example = [ + "--keep-daily 7" + "--keep-weekly 5" + "--keep-monthly 12" + "--keep-yearly 75" + ]; + }; }; })); default = {}; @@ -134,6 +151,11 @@ in let extraOptions = concatMapStrings (arg: " -o ${arg}") backup.extraOptions; resticCmd = "${pkgs.restic}/bin/restic${extraOptions}"; + pruneCmd = if (builtins.length backup.pruneOpts > 0) + then [ ( resticCmd + " forget --prune " + + (concatStringsSep " " backup.pruneOpts) ) + ( resticCmd + " check" ) ] + else []; in nameValuePair "restic-backups-${name}" ({ environment = { RESTIC_PASSWORD_FILE = backup.passwordFile; @@ -145,7 +167,7 @@ in restartIfChanged = false; serviceConfig = { Type = "oneshot"; - ExecStart = "${resticCmd} backup ${concatStringsSep " " backup.extraBackupArgs} ${concatStringsSep " " backup.paths}"; + ExecStart = [ "${resticCmd} backup ${concatStringsSep " " backup.extraBackupArgs} ${concatStringsSep " " backup.paths}" ] ++ pruneCmd; User = backup.user; } // optionalAttrs (backup.s3CredentialsFile != null) { EnvironmentFile = backup.s3CredentialsFile; From 1c9684abd6c6eca57e6fbfad01858523da355520 Mon Sep 17 00:00:00 2001 From: Matt McHenry Date: Sun, 5 Aug 2018 12:53:53 -0400 Subject: [PATCH 2/6] restic: add dynamicFilesFrom --- nixos/modules/services/backup/restic.nix | 41 +++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index a0af74f66997..4d35ed447f55 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -120,6 +120,17 @@ in "--keep-yearly 75" ]; }; + + dynamicFilesFrom = mkOption { + type = with types; nullOr str; + default = null; + description = '' + A script that produces a list of files to back up. The + results of this command are given to the '--files-from' + option. + ''; + example = "find /home/matt/git -type d -name .git"; + }; }; })); default = {}; @@ -151,6 +162,25 @@ in let extraOptions = concatMapStrings (arg: " -o ${arg}") backup.extraOptions; resticCmd = "${pkgs.restic}/bin/restic${extraOptions}"; + filesFromTmpFile = "/run/restic-backups-${name}/includes"; + preStartInit = if backup.initialize + then "${resticCmd} snapshots || ${resticCmd} init" + else ""; + dynamicFilesFromScript = pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom; + preStartFiles = if backup.dynamicFilesFrom != null + then "${dynamicFilesFromScript} > ${filesFromTmpFile}" + else ""; + preStartAttr = if (backup.initialize || backup.dynamicFilesFrom != null) + then { + preStart = '' + ${preStartInit} + ${preStartFiles} + ''; + } + else {}; + backupPaths = if (backup.dynamicFilesFrom == null) + then concatStringsSep " " backup.paths + else "--files-from ${filesFromTmpFile}"; pruneCmd = if (builtins.length backup.pruneOpts > 0) then [ ( resticCmd + " forget --prune " + (concatStringsSep " " backup.pruneOpts) ) @@ -167,14 +197,17 @@ in restartIfChanged = false; serviceConfig = { Type = "oneshot"; - ExecStart = [ "${resticCmd} backup ${concatStringsSep " " backup.extraBackupArgs} ${concatStringsSep " " backup.paths}" ] ++ pruneCmd; + ExecStart = [ "${resticCmd} backup ${concatStringsSep " " backup.extraBackupArgs} ${backupPaths}" ] ++ pruneCmd; User = backup.user; + RuntimeDirectory = "restic-backups-${name}"; } // optionalAttrs (backup.s3CredentialsFile != null) { EnvironmentFile = backup.s3CredentialsFile; }; - } // optionalAttrs backup.initialize { - preStart = '' - ${resticCmd} snapshots || ${resticCmd} init + } + // preStartAttr + // optionalAttrs (backup.dynamicFilesFrom != null) { + postStart = '' + rm ${filesFromTmpFile} ''; }) ) config.services.restic.backups; From 4fa2d4b5c3c8ea65d6b9c5c7d19a3c296714855e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 30 Jan 2020 17:07:21 +0000 Subject: [PATCH 3/6] nixos/restic: use optionalString/optionalAttrs where possible --- nixos/modules/services/backup/restic.nix | 41 +++++++++--------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index 4d35ed447f55..e1dccbba7632 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -163,37 +163,19 @@ in extraOptions = concatMapStrings (arg: " -o ${arg}") backup.extraOptions; resticCmd = "${pkgs.restic}/bin/restic${extraOptions}"; filesFromTmpFile = "/run/restic-backups-${name}/includes"; - preStartInit = if backup.initialize - then "${resticCmd} snapshots || ${resticCmd} init" - else ""; - dynamicFilesFromScript = pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom; - preStartFiles = if backup.dynamicFilesFrom != null - then "${dynamicFilesFromScript} > ${filesFromTmpFile}" - else ""; - preStartAttr = if (backup.initialize || backup.dynamicFilesFrom != null) - then { - preStart = '' - ${preStartInit} - ${preStartFiles} - ''; - } - else {}; backupPaths = if (backup.dynamicFilesFrom == null) then concatStringsSep " " backup.paths else "--files-from ${filesFromTmpFile}"; - pruneCmd = if (builtins.length backup.pruneOpts > 0) - then [ ( resticCmd + " forget --prune " + - (concatStringsSep " " backup.pruneOpts) ) - ( resticCmd + " check" ) ] - else []; + pruneCmd = optional (builtins.length backup.pruneOpts > 0) [ + ( resticCmd + " forget --prune " + (concatStringsSep " " backup.pruneOpts) ) + ( resticCmd + " check" ) + ]; in nameValuePair "restic-backups-${name}" ({ environment = { RESTIC_PASSWORD_FILE = backup.passwordFile; RESTIC_REPOSITORY = backup.repository; }; - path = with pkgs; [ - openssh - ]; + path = [ pkgs.openssh ]; restartIfChanged = false; serviceConfig = { Type = "oneshot"; @@ -203,9 +185,16 @@ in } // optionalAttrs (backup.s3CredentialsFile != null) { EnvironmentFile = backup.s3CredentialsFile; }; - } - // preStartAttr - // optionalAttrs (backup.dynamicFilesFrom != null) { + } // optionalAttrs (backup.initialize || backup.dynamicFilesFrom != null) { + preStart = '' + ${optionalString (backup.initialize) '' + ${resticCmd} snapshots || ${resticCmd} init + ''} + ${optionalString (backup.dynamicFilesFrom != null) '' + ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} > ${filesFromTmpFile} + ''} + ''; + } // optionalAttrs (backup.dynamicFilesFrom != null) { postStart = '' rm ${filesFromTmpFile} ''; From 5ad71cfe84a49fc27ebcf7c810bcced1361a6e6e Mon Sep 17 00:00:00 2001 From: Matt McHenry Date: Fri, 31 Jan 2020 22:06:19 -0500 Subject: [PATCH 4/6] fix pruneCmd to use optionals so multi-element list is preserved --- nixos/modules/services/backup/restic.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index e1dccbba7632..2388f1d6ca13 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -166,7 +166,7 @@ in backupPaths = if (backup.dynamicFilesFrom == null) then concatStringsSep " " backup.paths else "--files-from ${filesFromTmpFile}"; - pruneCmd = optional (builtins.length backup.pruneOpts > 0) [ + pruneCmd = optionals (builtins.length backup.pruneOpts > 0) [ ( resticCmd + " forget --prune " + (concatStringsSep " " backup.pruneOpts) ) ( resticCmd + " check" ) ]; From 42adda1ec4884e2118d59826f1294d4eadeab642 Mon Sep 17 00:00:00 2001 From: Bruno Bigras Date: Thu, 30 Jan 2020 16:31:52 -0500 Subject: [PATCH 5/6] nixos/tests/restic.nix: add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/restic.nix | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 nixos/tests/restic.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 89426865e1ac..e1ffe22e4b14 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -249,6 +249,7 @@ in radicale = handleTest ./radicale.nix {}; redis = handleTest ./redis.nix {}; redmine = handleTest ./redmine.nix {}; + restic = handleTest ./restic.nix {}; roundcube = handleTest ./roundcube.nix {}; rspamd = handleTest ./rspamd.nix {}; rss2email = handleTest ./rss2email.nix {}; diff --git a/nixos/tests/restic.nix b/nixos/tests/restic.nix new file mode 100644 index 000000000000..67bb7f1933d6 --- /dev/null +++ b/nixos/tests/restic.nix @@ -0,0 +1,63 @@ +import ./make-test-python.nix ( + { pkgs, ... }: + + let + password = "some_password"; + repository = "/tmp/restic-backup"; + passwordFile = pkgs.writeText "password" "correcthorsebatterystaple"; + in + { + name = "restic"; + + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ bbigras ]; + }; + + nodes = { + server = + { ... }: + { + services.restic.backups = { + remotebackup = { + inherit repository; + passwordFile = "${passwordFile}"; + initialize = true; + paths = [ "/opt" ]; + pruneOpts = [ + "--keep-daily 2" + "--keep-weekly 1" + "--keep-monthly 1" + "--keep-yearly 99" + ]; + }; + }; + }; + }; + + testScript = '' + server.start() + server.wait_for_unit("dbus.socket") + server.fail( + "${pkgs.restic}/bin/restic -r ${repository} -p ${passwordFile} snapshots" + ) + server.succeed( + "mkdir -p /opt", + "touch /opt/some_file", + "timedatectl set-time '2016-12-13 13:45'", + "systemctl start restic-backups-remotebackup.service", + '${pkgs.restic}/bin/restic -r ${repository} -p ${passwordFile} snapshots -c | grep -e "^1 snapshot"', + "timedatectl set-time '2017-12-13 13:45'", + "systemctl start restic-backups-remotebackup.service", + "timedatectl set-time '2018-12-13 13:45'", + "systemctl start restic-backups-remotebackup.service", + "timedatectl set-time '2018-12-14 13:45'", + "systemctl start restic-backups-remotebackup.service", + "timedatectl set-time '2018-12-15 13:45'", + "systemctl start restic-backups-remotebackup.service", + "timedatectl set-time '2018-12-16 13:45'", + "systemctl start restic-backups-remotebackup.service", + '${pkgs.restic}/bin/restic -r ${repository} -p ${passwordFile} snapshots -c | grep -e "^4 snapshot"', + ) + ''; + } +) From 447207114fa11f37b5b51eb8526b9530695b9c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 7 Feb 2020 10:39:18 +0000 Subject: [PATCH 6/6] restic: reference nixos test --- pkgs/tools/backup/restic/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/tools/backup/restic/default.nix b/pkgs/tools/backup/restic/default.nix index 26f05d419545..1eb02e9a4834 100644 --- a/pkgs/tools/backup/restic/default.nix +++ b/pkgs/tools/backup/restic/default.nix @@ -1,4 +1,4 @@ -{ lib, buildGoPackage, fetchFromGitHub }: +{ lib, buildGoPackage, fetchFromGitHub, nixosTests }: buildGoPackage rec { pname = "restic"; @@ -18,6 +18,8 @@ buildGoPackage rec { go run build.go ''; + passthru.tests.restic = nixosTests.restic; + installPhase = '' mkdir -p \ $bin/bin \