Add support for PG17

In PostgreSQL versions prior to 17, the `pg_walfile_name` function could
return the previous WAL file if the provided LSN fell at the boundary between WAL segments.
This was fixed in PG17, and now gets the actual wal file name instead. This change affects
the way we are retrieving the backup_end_wal_segment for the backups metadata.
Therefore, instead of relying on our old behavior, we are now using the LSN yield by
pg_backup_stop.

.github/workflows/build.yml

@@ -77,7 +77,7 @@ jobs:
           wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
           sudo apt-get update
           # Setup build deps
-          sudo apt-get install -y libsnappy-dev postgresql-10 postgresql-11 postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16
+          sudo apt-get install -y libsnappy-dev postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16 postgresql-17
           # Setup common python dependencies
           python -m pip install --upgrade pip
           pip install .

README.rst

@@ -133,7 +133,7 @@ python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql
 By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix
 ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command.
 
-Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16)
+Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17)
 
 Bring up vagrant instance and connect via ssh::
 

Vagrantfile

@@ -39,7 +39,7 @@ Vagrant.configure("2") do |config|
         sed -i "s/^#create_main_cluster.*/create_main_cluster=false/g" /etc/postgresql-common/createcluster.conf
 
         apt-get install -y python{3.10,3.11,3.12} python{3.10,3.11,3.12}-dev python{3.10,3.11,3.12}-venv
-        apt-get install -y postgresql-{12,13,14,15,16} postgresql-server-dev-{12,13,14,15,16}
+        apt-get install -y postgresql-{12,13,14,15,16,17} postgresql-server-dev-{12,13,14,15,16,17}
 
         username="$(< /dev/urandom tr -dc a-z | head -c${1:-32};echo;)"
         password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;)

docs/development.rst

@@ -54,12 +54,12 @@ Vagrant
 =======
 
 The Vagrantfile can be used to setup a vagrant development environment.   The vagrant environment has
-python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15 and 16.
+python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15, 16 and 17.
 
 By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix
 ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command.
 
-Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16)
+Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15, 16 and 17)
 
 Bring up vagrant instance and connect via ssh::
 

pghoard/config.py

@@ -17,7 +17,7 @@
 from pghoard.common import (extract_pg_command_version_string, pg_major_version, pg_version_string_to_number)
 from pghoard.postgres_command import PGHOARD_HOST, PGHOARD_PORT
 
-SUPPORTED_VERSIONS = ["16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"]
+SUPPORTED_VERSIONS = ["17", "16", "15", "14", "13", "12", "11", "10", "9.6", "9.5", "9.4", "9.3"]
 
 
 def get_cpu_count():

pghoard/wal.py

@@ -32,6 +32,7 @@
     0xD10D: 140000,
     0xD110: 150000,
     0xD113: 160000,
+    0xD116: 170000,
 }
 WAL_MAGIC_BY_VERSION = {value: key for key, value in WAL_MAGIC.items()}
 

test/conftest.py

@@ -35,7 +35,7 @@
 
 logutil.configure_logging()
 
-DEFAULT_PG_VERSIONS = ["16", "15", "14", "13", "12"]
+DEFAULT_PG_VERSIONS = ["17", "16", "15", "14", "13", "12"]
 
 
 def port_is_listening(hostname: str, port: int, timeout: float = 0.5) -> bool:

test/test_webserver.py

@@ -242,7 +242,7 @@ def _switch_wal(self, db, count):
         conn.close()
         return start_wal, end_wal
 
-    def test_archive_sync(self, db, pghoard):
+    def test_archive_sync(self, db, pghoard, pg_version: str):
         log = logging.getLogger("test_archive_sync")
         store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
 
@@ -273,7 +273,11 @@ def list_archive(folder):
         # cluster between all tests)
         pg_wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
         pg_wals = {f for f in os.listdir(pg_wal_dir) if wal.WAL_RE.match(f) and f > start_wal}
-        assert len(pg_wals) >= 4
+
+        # consider changes in pg_walfile_name, as pg_walfile_name(pg_current_wal_lsn()) might return
+        # previous walfile name and not current.
+        expected_min_wals = 4 if int(pg_version) < 17 else 3
+        assert len(pg_wals) >= expected_min_wals
 
         # create a couple of "recycled" xlog files that we must ignore
         last_wal = sorted(pg_wals)[-1]
@@ -291,7 +295,7 @@ def write_dummy_wal(inc):
         # check what we have archived, there should be at least the three
         # above WALs that are NOT there at the moment
         archived_wals = set(list_archive("xlog"))
-        assert len(pg_wals - archived_wals) >= 4
+        assert len(pg_wals - archived_wals) >= expected_min_wals
         # now perform an archive sync
         arsy = ArchiveSync()
         arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
@@ -329,7 +333,7 @@ def write_dummy_wal(inc):
             "restore_command = 'false'",
         ]
         if Version(db.pgver).major >= 12:
-            with open(os.path.join(db.pgdata, "standby.signal"), "w") as fp:
+            with open(os.path.join(db.pgdata, "standby.signal"), "w"):
                 pass
 
             recovery_conf_path = "postgresql.auto.conf"
@@ -339,7 +343,7 @@ def write_dummy_wal(inc):
             recovery_conf_path = "recovery.conf"
             open_mode = "w"
 
-        with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp:
+        with open(os.path.join(db.pgdata, recovery_conf_path), open_mode) as fp:  # type: ignore
             fp.write("\n".join(recovery_conf) + "\n")
 
         # start PG and promote it