feat(nix): refresh glibc collation version on postgres start

ansible/files/postgresql_config/postgresql.service

@@ -1,6 +1,7 @@
 [Unit]
 Description=PostgreSQL database server
 Documentation=man:postgres(1)
+Wants=postgresql-collation-refresh.service
 Requires=database-optimizations.service
 After=database-optimizations.service
 

ansible/tasks/setup-postgres.yml

@@ -246,6 +246,13 @@
   loop_control:
     loop_var: 'systemd_svc_item'
 
+- name: Link the collation refresh unit from the site profile
+  when: stage2
+  ansible.builtin.file:
+    src: /nix/var/nix/profiles/site/lib/systemd/system/postgresql-collation-refresh.service
+    dest: /etc/systemd/system/postgresql-collation-refresh.service
+    state: link
+
 - name: initialize pg required state
   ansible.builtin.file:
     group: 'postgres'

nix/ext/tests/collation-refresh.nix

@@ -0,0 +1,100 @@
+{ self, pkgs }:
+let
+  system = pkgs.pkgsLinux.stdenv.hostPlatform.system;
+  testLib = import ./lib.nix { inherit self pkgs; };
+  refresh = self.packages.${system}.postgres-collation-refresh;
+  psql = self.packages.${system}."psql_17/bin";
+in
+pkgs.testers.runNixOSTest {
+  name = "collation-refresh";
+  nodes.server =
+    { ... }:
+    {
+      imports = [
+        (testLib.makeSupabaseTestConfig {
+          majorVersion = "17";
+        })
+      ];
+
+      i18n.supportedLocales = [
+        "C.UTF-8/UTF-8"
+        "en_US.UTF-8/UTF-8"
+        "de_DE.UTF-8/UTF-8"
+      ];
+
+      systemd.tmpfiles.rules = [
+        "L+ /var/lib/postgresql/.nix-profile - - - - ${psql}"
+        "L+ /nix/var/nix/profiles/site - - - - ${refresh}"
+      ];
+      systemd.packages = [ refresh ];
+      systemd.services.postgresql.wants = [ "postgresql-collation-refresh.service" ];
+    };
+  testScript = ''
+    PSQL = "psql -h localhost -U supabase_admin -d postgres -At -v ON_ERROR_STOP=1"
+    UNIT = "postgresql-collation-refresh.service"
+
+    def sql(query):
+      return server.succeed(PSQL + " -c \"" + query + "\"").strip()
+
+    def recorded(db):
+      return sql("select coalesce(datcollversion, '-') from pg_database where datname = '" + db + "'")
+
+    def set_recorded(db, version):
+      sql("update pg_database set datcollversion = '" + version + "' where datname = '" + db + "'")
+
+    start_all()
+    server.wait_for_unit("supabase-db-init.service")
+    server.wait_for_unit(UNIT)
+
+    DBS = {
+      "g": "libc locale = 'en_US.UTF-8'",
+      "out": "libc locale = 'en_US.UTF-8'",
+      "de": "libc locale = 'de_DE.UTF-8'",
+      "c": "libc locale = 'C'",
+      "i": "icu locale = 'en_US.UTF-8' icu_locale = 'en-US'",
+    }
+    for db, options in DBS.items():
+      sql("create database " + db + " locale_provider = " + options + " template template0")
+    actual = recorded("g")
+    fresh = {"g": actual, "out": actual, "de": actual, "i": recorded("i")}
+
+    def reset(stale):
+      for db, version in {**fresh, **stale}.items():
+        set_recorded(db, version)
+
+    def check(title, stale, refreshed, fails):
+      with subtest(title):
+        reset(stale)
+        status, _ = server.execute("systemctl restart " + UNIT)
+        assert (status != 0) == fails, f"unit status {status}, expected failure: {fails}"
+        for db, version in stale.items():
+          expected = actual if db in refreshed else version
+          assert recorded(db) == expected, f"{db}: expected {expected}, got {recorded(db)}"
+        assert recorded("c") == "-"
+
+    check("Nothing stale", {}, [], False)
+    check("A stale glibc en_US database inside the bounds is refreshed", {"g": "2.31"}, ["g"], False)
+    check("ICU and non-English glibc databases are left alone", {"de": "2.31", "i": "1.0"}, [], False)
+    check("A database above 2.44 fails the unit, the others are refreshed", {"out": "2.99", "g": "2.31"}, ["g"], True)
+    check("A database below 2.31 fails the unit", {"out": "2.28"}, [], True)
+
+    with subtest("A database that fails to refresh does not stop the others"):
+      sql("create role refresher login")
+      sql("create database a_other locale_provider = libc locale = 'en_US.UTF-8' template template0")
+      sql("create database b_owned owner refresher locale_provider = libc locale = 'en_US.UTF-8' template template0")
+      reset({"a_other": "2.31", "b_owned": "2.31"})
+      status, _ = server.execute("psql -h localhost -U refresher -d postgres -X -q -f /nix/var/nix/profiles/site/share/postgres-collation-refresh.sql")
+      assert status != 0
+      assert recorded("a_other") == "2.31"
+      assert recorded("b_owned") == actual
+
+    with subtest("A postgres restart reruns the unit and leaves postgres active when it fails"):
+      reset({"out": "2.99", "g": "2.31", "a_other": actual})
+      server.succeed("systemctl reset-failed " + UNIT)
+      server.succeed("systemctl restart postgresql")
+      server.wait_until_succeeds("systemctl is-failed " + UNIT)
+      server.succeed("systemctl is-active postgresql.service")
+      assert recorded("g") == actual
+      assert recorded("out") == "2.99"
+  '';
+}

nix/packages/default.nix

@@ -61,6 +61,7 @@
           };
           docs = pkgs.callPackage ./docs.nix { };
           pgbouncer = pkgs.callPackage ../pgbouncer.nix { };
+          postgres-collation-refresh = pkgs.callPackage ./postgres-collation-refresh { };
           github-matrix = pkgs.callPackage ./github-matrix {
             nix-eval-jobs = inputs'.nix-eval-jobs.packages.default;
           };

nix/packages/postgres-collation-refresh/default.nix

@@ -0,0 +1,22 @@
+{
+  lib,
+  stdenv,
+  runCommand,
+  replaceVars,
+}:
+let
+  minGlibc = "2.31";
+  maxGlibc = "2.44";
+in
+assert lib.assertMsg
+  (
+    !stdenv.hostPlatform.isGnu
+    || lib.versionAtLeast maxGlibc (lib.versions.majorMinor stdenv.cc.libc.version)
+  )
+  "glibc ${stdenv.cc.libc.version} is above the reviewed maximum ${maxGlibc}. Review English collation changes, then raise maxGlibc in nix/packages/postgres-collation-refresh/default.nix.";
+runCommand "postgres-collation-refresh" { } ''
+  install -Dm0644 ${
+    replaceVars ./postgres-collation-refresh.sql { inherit minGlibc maxGlibc; }
+  } $out/share/postgres-collation-refresh.sql
+  install -Dm0644 ${./postgresql-collation-refresh.service} $out/lib/systemd/system/postgresql-collation-refresh.service
+''

nix/packages/postgres-collation-refresh/postgres-collation-refresh.sql

@@ -0,0 +1,28 @@
+select pg_is_in_recovery() as replica \gset
+\if :replica
+\quit
+\endif
+
+create temp view stale as
+select datname,
+       string_to_array(datcollversion, '.')::int[] between string_to_array('@minGlibc@', '.')::int[] and string_to_array('@maxGlibc@', '.')::int[]
+       and string_to_array(pg_database_collation_actual_version(oid), '.')::int[] between string_to_array('@minGlibc@', '.')::int[] and string_to_array('@maxGlibc@', '.')::int[] as in_bounds
+from pg_database
+where datlocprovider = 'c'
+  and datcollate in ('en_US.UTF-8', 'en_US.utf8')
+  and datcollversion is not null
+  and datcollversion <> pg_database_collation_actual_version(oid);
+
+\set ON_ERROR_STOP off
+select format('alter database %I refresh collation version', datname) from stale where in_bounds order by datname
+\gexec
+
+\set ON_ERROR_STOP on
+do $$
+declare names text;
+begin
+  select string_agg(format('%I', datname), ', ' order by datname) into names from stale;
+  if names is not null then
+    raise exception 'collation_refresh: still stale: %', names;
+  end if;
+end $$;

nix/packages/postgres-collation-refresh/postgresql-collation-refresh.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Refresh default collation version
+After=postgresql.service
+PartOf=postgresql.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+User=postgres
+TimeoutStartSec=120
+ExecStart=/var/lib/postgresql/.nix-profile/bin/psql -X -q -h localhost -U supabase_admin -d postgres -f /nix/var/nix/profiles/site/share/postgres-collation-refresh.sql

nix/packages/site-env.nix

@@ -39,6 +39,7 @@
           paths = [
             supautils
             activate
+            self'.packages.postgres-collation-refresh
           ]
           ++ extraPaths;
           passthru = { inherit activate; };

testinfra/test_ami_nix.py

@@ -1616,6 +1616,16 @@ def test_postgres_prestart_does_not_reset_core_limit(host):
     )
 
 
+def test_collation_refresh_unit_is_linked_and_not_failed(host):
+    """The collation refresh unit links from the site profile and is active."""
+    unit = "postgresql-collation-refresh.service"
+    link = run_ssh_command(host["ssh"], f"readlink /etc/systemd/system/{unit}")
+    assert link["stdout"].startswith("/nix/var/nix/profiles/site/"), link
+    state = run_ssh_command(host["ssh"], f"systemctl is-active {unit}")
+    journal = run_ssh_command(host["ssh"], f"journalctl -u {unit} --no-pager -n 40")
+    assert state["stdout"].strip() == "active", journal["stdout"]
+
+
 def _crash_a_backend_and_wait_for_recovery(host):
     """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's
     normal crash-recovery to bring the instance back on its own. Returns the