From 9656e1aa9d466b83157954d2bc89835ac2cce0c3 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Fri, 9 May 2025 14:36:01 +0200 Subject: [PATCH] nixos/postgresql: make postgresql.target wait until recovery is done The new postgresql.target will now wait until recovery is done and read/write connections are possible. This allows ensure* scripts and downstream migrations to work properly after recovery from backup. Resolves #346886 --- nixos/doc/manual/release-notes/rl-2511.section.md | 2 +- nixos/modules/services/databases/postgresql.md | 8 ++++++++ nixos/modules/services/databases/postgresql.nix | 10 +++++++++- nixos/tests/pgbackrest/posix.nix | 9 ++++++++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2511.section.md b/nixos/doc/manual/release-notes/rl-2511.section.md index 80566d9ddc02..66baf0b0f79b 100644 --- a/nixos/doc/manual/release-notes/rl-2511.section.md +++ b/nixos/doc/manual/release-notes/rl-2511.section.md @@ -64,7 +64,7 @@ - The `yeahwm` package and `services.xserver.windowManager.yeahwm` module were removed due to the package being broken and unmaintained upstream. -- The `services.postgresql` module now sets up a systemd unit `postgresql.target`. Depending on `postgresql.target` guarantees that initial/ensure scripts were executed. +- The `services.postgresql` module now sets up a systemd unit `postgresql.target`. Depending on `postgresql.target` guarantees that postgres is in read-write mode and initial/ensure scripts were executed. Depending on `postgresql.service` only guarantees a read-only connection. - The `services.siproxd` module has been removed as `siproxd` is unmaintained and broken with libosip 5.x. diff --git a/nixos/modules/services/databases/postgresql.md b/nixos/modules/services/databases/postgresql.md index 150a90f34356..aa38ebb3a249 100644 --- a/nixos/modules/services/databases/postgresql.md +++ b/nixos/modules/services/databases/postgresql.md @@ -105,6 +105,14 @@ database migrations. #### in intermediate oneshot service {#module-services-postgres-initializing-extra-permissions-superuser-oneshot} +Make sure to run this service after `postgresql.target`, not `postgresql.service`. + +They differ in two aspects: +- `postgresql.target` includes `postgresql-setup`, so users managed via `ensureUsers` are already created. +- `postgresql.target` will wait until PostgreSQL is in read-write mode after restoring from backup, while `postgresql.service` will already be ready when PostgreSQL is still recovering in read-only mode. + +Both can lead to unexpected errors either during initial database creation or restore, when using `postgresql.service`. + ```nix { systemd.services."migrate-service1-db1" = { diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 96c718b9babb..29cef9b40e18 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -879,7 +879,15 @@ in # Wait for PostgreSQL to be ready to accept connections. script = '' - while ! psql -d postgres -c "" 2> /dev/null; do + check-connection() { + psql -d postgres -v ON_ERROR_STOP=1 <<-' EOF' + SELECT pg_is_in_recovery() \gset + \if :pg_is_in_recovery + \i still-recovering + \endif + EOF + } + while ! check-connection 2> /dev/null; do if ! systemctl is-active --quiet postgresql.service; then exit 1; fi sleep 0.1 done diff --git a/nixos/tests/pgbackrest/posix.nix b/nixos/tests/pgbackrest/posix.nix index d5bfaa6d310c..aede13ff5cf8 100644 --- a/nixos/tests/pgbackrest/posix.nix +++ b/nixos/tests/pgbackrest/posix.nix @@ -27,6 +27,13 @@ in CREATE TABLE t(c text); INSERT INTO t VALUES ('hello world'); ''; + # To make sure we're waiting for read-write after recovery. + ensureUsers = [ + { + name = "app-user"; + ensureClauses.login = true; + } + ]; }; services.pgbackrest = { @@ -141,7 +148,7 @@ in primary.succeed("sudo -u postgres pgbackrest --stanza=default restore --delta") primary.systemctl("start postgresql") - primary.wait_for_unit("postgresql.service") + primary.wait_for_unit("postgresql.target") assert "hello world" in primary.succeed("sudo -u postgres psql -c 'TABLE t;'") ''; }