From 03d0fed6f89352200d32c455b50844a616d997d0 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 26 Jul 2025 19:09:48 +0200 Subject: [PATCH 1/2] nixos/postgresql: implement auto-restart & rework dependencies of postgresql.target At my employer's NixOS-based platform, PostgreSQL is configured with `Restart=always` which got never upstreamed, unfortunately. This however revealed an interesting problem when using bi-directional BindsTo: when killing `postgresql.service`, sometimes both the service & target starts back up and sometimes they don't. According to an upstream bugreport[1] this is a known problem because you have two conflicting operations scheduled in a single transaction, namely * When (auto-)restarting, a restart job for all units bound to the restarting unit are immediately scheduled[2]. * Due to the `BindsTo` relationship, a stop-job for `postgresql.target` is scheduled immediately by the manager loop[3]. This is caused by the `UNIT_ATOM_CANNOT_BE_ACTIVE_WITHOUT` "atom" which is ONLY set for a BindsTo relationship[4]. When this is processed first, the restart is inhibited: Jul 12 13:25:51 nixos systemd[1]: postgresql.service: Main process exited, code=killed, status=9/KILL Jul 12 13:25:51 nixos systemd[1]: postgresql.service: Changed running -> stop-sigterm Jul 12 13:25:51 nixos systemd[1]: postgresql.target: Trying to enqueue job postgresql.target/stop/replace Jul 12 13:25:51 nixos systemd[1]: postgresql.service: Installed new job postgresql.service/stop as 80053 Jul 12 13:25:51 nixos systemd[1]: postgresql.target: Installed new job postgresql.target/stop as 80052 Jul 12 13:25:51 nixos systemd[1]: postgresql.target: Enqueued job postgresql.target/stop as 80052 [...] Jul 12 13:25:51 nixos systemd[1]: postgresql.service: Service restart not allowed. It's subtle and non-obvious from the man-page, but the way how units are stopped is different when using `PartOf=` or `Requires=` which don't have the `UNIT_ATOM_CANNOT_BE_ACTIVE_WITHOUT` property, but instead schedules the stop/start of the target AFTER the stop-job of postgresql.service which is turned into a start-job because of Restart=always: Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Main process exited, code=killed, status=9/KILL [...] Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Failed with result 'signal'. Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Service will restart (restart setting) [...] Jul 12 13:33:00 nixos systemd[1]: postgresql.target: Installed new job postgresql.target/restart as 80996 Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Installed new job postgresql.service/restart as 80907 [...] Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Scheduled restart job, restart counter is at 1. [...] Jul 12 13:33:00 nixos systemd[1]: Stopped target postgresql.target. Jul 12 13:33:00 nixos systemd[1]: postgresql.target: Converting job postgresql.target/restart -> postgresql.target/start Jul 12 13:33:00 nixos systemd[1]: Stopping postgresql.target... [...] Jul 12 13:33:00 nixos systemd[1]: Stopped postgresql.service. Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Converting job postgresql.service/restart -> postgresql.service/start [...] Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Changed dead -> running Jul 12 13:33:00 nixos systemd[1]: postgresql.service: Job 80907 postgresql.service/start finished, result=done Jul 12 13:33:00 nixos systemd[1]: Started postgresql.service. Jul 12 13:33:00 nixos systemd[1]: postgresql.target: Changed dead -> active [...] Jul 12 13:33:00 nixos systemd[1]: Reached target postgresql.target. Do note that the stop job (including the restart) of postgresql.service is fully processed here before dealing with PartOf/ConsistsOf relationships. I tested this against the following cases: | Unit | Action | Propagates to | | ------------------ | ------------ | ------------------ | | postgresql.target | restart | postgresql.service | | postgresql.target | start | postgresql.service | | postgresql.target | stop | psotgresql.service | | postgresql.service | start | postgresql.target | | postgresql.service | restart | postgresql.target | | postgresql.service | stop | postgresql.target | | postgresql.service | auto-restart | postgresql.target | | postgresql.service | failure | postgresql.target | [1] e.g. systemd issue 8374 [2] https://github.com/systemd/systemd/blob/v256-stable/src/core/service.c#L2535-L2542 [3] https://github.com/systemd/systemd/blob/v256-stable/src/core/manager.c#L1611-L1626 [4] https://github.com/systemd/systemd/blob/v256-stable/src/core/unit-dependency-atom.c#L30-L35 --- nixos/modules/services/databases/postgresql.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index d1fd9797e4d4..f22d18aabe84 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -769,7 +769,7 @@ in systemd.targets.postgresql = { description = "PostgreSQL"; wantedBy = [ "multi-user.target" ]; - bindsTo = [ + requires = [ "postgresql.service" "postgresql-setup.service" ]; @@ -780,8 +780,13 @@ in after = [ "network.target" ]; - # To trigger the .target also on "systemctl start postgresql". - bindsTo = [ "postgresql.target" ]; + # To trigger the .target also on "systemctl start postgresql" as well as on + # restarts & stops. + # Please note that postgresql.service & postgresql.target binding to + # each other makes the Restart=always rule racy and results + # in sometimes the service not being restarted. + wants = [ "postgresql.target" ]; + partOf = [ "postgresql.target" ]; environment.PGDATA = cfg.dataDir; From 6ae194e419471d9f6afe98e988cc2be172f72479 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 26 Jul 2025 19:11:54 +0200 Subject: [PATCH 2/2] nixos/postgresql: set Restart=always for postgresql.service ...including a slightly more careful config around restarts, i.e. * We have intervals of 5 seconds between restarts instead of 100ms. * If we exceed 5 start attempts in 5*120s (with 120s being the timeout), start job gets rate-limited and thus aborted. Do note that there are at most 5 start attempts allowed in ~625s by default. If the startup fails very quickly, either wait until the rate-limit is over or reset the counter using `systemctl reset-failed postgresql.service`. * The interval of 625s (plus 5s of buffer) are automatically derived from RestartSec & TimeoutSec. Changing either will also affect StartLimitIntervalSec unless overridden with `mkForce`. --- nixos/modules/services/databases/postgresql.nix | 17 ++++++++++++++++- nixos/tests/postgresql/postgresql.nix | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index f22d18aabe84..84a99bcb329b 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -826,6 +826,8 @@ in ExecStart = "${cfg.finalPackage}/bin/postgres"; + Restart = "always"; + # Hardening CapabilityBoundingSet = [ "" ]; DevicePolicy = "closed"; @@ -877,7 +879,20 @@ in }) ]; - unitConfig.RequiresMountsFor = "${cfg.dataDir}"; + unitConfig = + let + inherit (config.systemd.services.postgresql.serviceConfig) TimeoutSec; + maxTries = 5; + bufferSec = 5; + in + { + RequiresMountsFor = "${cfg.dataDir}"; + + # The max. time needed to perform `maxTries` start attempts of systemd + # plus a bit of buffer time (bufferSec) on top. + StartLimitIntervalSec = TimeoutSec * maxTries + bufferSec; + StartLimitBurst = maxTries; + }; }; systemd.services.postgresql-setup = { diff --git a/nixos/tests/postgresql/postgresql.nix b/nixos/tests/postgresql/postgresql.nix index e77ec7897547..456cacb54830 100644 --- a/nixos/tests/postgresql/postgresql.nix +++ b/nixos/tests/postgresql/postgresql.nix @@ -101,6 +101,12 @@ let machine.fail(check_count("SELECT * FROM sth;", 4)) machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) + with subtest("killing postgres process should trigger an automatic restart"): + machine.succeed("systemctl kill -s KILL postgresql") + + machine.wait_until_succeeds("systemctl is-active postgresql.service") + machine.wait_until_succeeds("systemctl is-active postgresql.target") + with subtest("Backup service works"): machine.succeed( "systemctl start ${backupService}.service",