From 0777605832977ea797577c1f2d2265e363b6763a Mon Sep 17 00:00:00 2001 From: Ahmed Et-tanany Date: Mon, 17 Jun 2024 12:56:03 +0200 Subject: [PATCH] Update supported python and postgres versions --- .github/workflows/build.yml | 8 ++++---- README.rst | 22 ++++++++-------------- Vagrantfile | 10 +++++----- docs/development.rst | 18 ++++++------------ pghoard/common.py | 4 ++-- pghoard/restore.py | 14 +++++++------- pyproject.toml | 3 +-- test/base.py | 4 ++-- test/conftest.py | 6 +++--- test/test_webserver.py | 4 ++-- 10 files changed, 40 insertions(+), 53 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ef8ff23..1ad5ca73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,11 +14,11 @@ on: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: # only use one version for the lint step - python-version: [3.9] + python-version: [3.10] steps: @@ -54,12 +54,12 @@ jobs: fi test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: lint strategy: max-parallel: 5 matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - id: checkout-code diff --git a/README.rst b/README.rst index 4f7fa941..242244b5 100644 --- a/README.rst +++ b/README.rst @@ -81,10 +81,10 @@ Requirements ============ PGHoard can backup and restore PostgreSQL versions 9.6 and above, but is -only tested and actively developed with version 10 and above. +only tested and actively developed with version 12 and above. The daemon is implemented in Python and is tested and developed with version -3.8 and above. The following Python modules are required: +3.10 and above. The following Python modules are required: * psycopg2_ to look up transaction log metadata * requests_ for the internal client-server architecture @@ -128,12 +128,12 @@ Vagrant ======= The Vagrantfile can be used to setup a vagrant development environment. The vagrant environment has -python 3.8, 3.9 and 3.10 virtual environments and installations of postgresql 10, 11 and 12, 13 and 14. +python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15 and 16. By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command. -Any combination of Python (3.8, 3.9 and 3.10) and Postgresql (10, 11, 12, 13 and 14) +Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16) Bring up vagrant instance and connect via ssh:: @@ -141,21 +141,15 @@ Bring up vagrant instance and connect via ssh:: vagrant ssh vagrant@ubuntu2004:~$ cd /vagrant -Test with Python 3.8 and Postgresql 11:: +Test with Python 3.11 and Postgresql 12:: - vagrant@ubuntu2004:~$ source ~/venv3.8/bin/activate - vagrant@ubuntu2004:~$ PG_VERSION=11 make unittest - vagrant@ubuntu2004:~$ deactivate - -Test with Python 3.9 and Postgresql 12:: - - vagrant@ubuntu2004:~$ source ~/venv3.9/bin/activate + vagrant@ubuntu2004:~$ source ~/venv3.11/bin/activate vagrant@ubuntu2004:~$ PG_VERSION=12 make unittest vagrant@ubuntu2004:~$ deactivate -Test with Python 3.10 and Postgresql 13:: +Test with Python 3.12 and Postgresql 13:: - vagrant@ubuntu2004:~$ source ~/venv3.10/bin/activate + vagrant@ubuntu2004:~$ source ~/venv3.12/bin/activate vagrant@ubuntu2004:~$ PG_VERSION=13 make unittest vagrant@ubuntu2004:~$ deactivate diff --git a/Vagrantfile b/Vagrantfile index 010fbfe2..5a7d6338 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -38,8 +38,8 @@ Vagrant.configure("2") do |config| sed -i "s/^#start_conf.*/start_conf='manual'/g" /etc/postgresql-common/createcluster.conf sed -i "s/^#create_main_cluster.*/create_main_cluster=false/g" /etc/postgresql-common/createcluster.conf - apt-get install -y python{3.8,3.9,3.10} python{3.8,3.9,3.10}-dev python{3.8,3.9,3.10}-venv - apt-get install -y postgresql-{11,12,13,14,15,16} postgresql-server-dev-{11,12,13,14,15,16} + apt-get install -y python{3.10,3.11,3.12} python{3.10,3.11,3.12}-dev python{3.10,3.11,3.12}-venv + apt-get install -y postgresql-{12,13,14,15,16} postgresql-server-dev-{12,13,14,15,16} username="$(< /dev/urandom tr -dc a-z | head -c${1:-32};echo;)" password=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo;) @@ -68,13 +68,13 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: $script, privileged: true $script = <<-SCRIPT - versions=(3.8 3.9 3.10) + versions=(3.10 3.11 3.12) for version in "${versions[@]}"; do python${version} -m venv venv${version} source ~/venv${version}/bin/activate pip install --upgrade pip - pip install -r /vagrant/requirements.txt - pip install --upgrade -r /vagrant/requirements.dev.txt + pip install "/vagrant/." + pip install --upgrade "/vagrant/.[dev]" done SCRIPT config.vm.provision "shell", inline: $script, privileged: false diff --git a/docs/development.rst b/docs/development.rst index 01f6b286..576e01ec 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -54,12 +54,12 @@ Vagrant ======= The Vagrantfile can be used to setup a vagrant development environment. The vagrant environment has -python 3.8, 3.9 and 3.10 virtual environments and installations of postgresql 10, 11 and 12, 13 and 14. +python 3.10, 3.11 and 3.12 virtual environments and installations of postgresql 12, 13, 14, 15 and 16. By default vagrant up will start a Virtualbox environment. The Vagrantfile will also work for libvirt, just prefix ``VAGRANT_DEFAULT_PROVIDER=libvirt`` to the ``vagrant up`` command. -Any combination of Python (3.8, 3.9 and 3.10) and Postgresql (10, 11, 12, 13 and 14) +Any combination of Python (3.10, 3.11 and 3.12) and Postgresql (12, 13, 14, 15 and 16) Bring up vagrant instance and connect via ssh:: @@ -67,21 +67,15 @@ Bring up vagrant instance and connect via ssh:: vagrant ssh vagrant@ubuntu2004:~$ cd /vagrant -Test with Python 3.8 and Postgresql 11:: +Test with Python 3.11 and Postgresql 12:: - vagrant@ubuntu2004:~$ source ~/venv3.8/bin/activate - vagrant@ubuntu2004:~$ PG_VERSION=11 make unittest - vagrant@ubuntu2004:~$ deactivate - -Test with Python 3.9 and Postgresql 12:: - - vagrant@ubuntu2004:~$ source ~/venv3.9/bin/activate + vagrant@ubuntu2004:~$ source ~/venv3.11/bin/activate vagrant@ubuntu2004:~$ PG_VERSION=12 make unittest vagrant@ubuntu2004:~$ deactivate -Test with Python 3.10 and Postgresql 13:: +Test with Python 3.12 and Postgresql 13:: - vagrant@ubuntu2004:~$ source ~/venv3.10/bin/activate + vagrant@ubuntu2004:~$ source ~/venv3.12/bin/activate vagrant@ubuntu2004:~$ PG_VERSION=13 make unittest vagrant@ubuntu2004:~$ deactivate diff --git a/pghoard/common.py b/pghoard/common.py index 27416111..111e1142 100644 --- a/pghoard/common.py +++ b/pghoard/common.py @@ -19,12 +19,12 @@ import time from contextlib import suppress from dataclasses import dataclass, field -from distutils.version import LooseVersion from pathlib import Path from queue import Queue from threading import Thread from typing import (TYPE_CHECKING, Any, BinaryIO, Callable, Dict, Final, Optional, Protocol, Tuple, cast) +from packaging.version import Version from pydantic import BaseModel, Field from rohmu import IO_BLOCK_SIZE, BaseTransfer, rohmufile from rohmu.errors import Error, InvalidConfigurationError @@ -415,7 +415,7 @@ def extract_pghoard_delta_metadata(fileobj: FileLike) -> Dict[str, Any]: def get_pg_wal_directory(config): - if LooseVersion(config["pg_data_directory_version"]) >= "10": + if Version(config["pg_data_directory_version"]).major >= 10: return os.path.join(config["pg_data_directory"], "pg_wal") return os.path.join(config["pg_data_directory"], "pg_xlog") diff --git a/pghoard/restore.py b/pghoard/restore.py index 9c9cb91a..c4050a10 100644 --- a/pghoard/restore.py +++ b/pghoard/restore.py @@ -24,11 +24,10 @@ import uuid # ignore pylint/distutils issue, https://github.com/PyCQA/pylint/issues/73 from dataclasses import dataclass, field -from distutils.version import \ - LooseVersion # pylint: disable=no-name-in-module,import-error from threading import RLock from typing import Any, Dict, List, Optional, Set, Union +from packaging.version import Version from psycopg2.extensions import adapt from rohmu import dates, get_transfer, rohmufile from rohmu.errors import (Error, InvalidConfigurationError, MaybeRecoverableError) @@ -115,12 +114,13 @@ def create_recovery_conf( "%f", ] with open(os.path.join(dirpath, "PG_VERSION"), "r") as fp: - pg_version = LooseVersion(fp.read().strip()) + v = Version(fp.read().strip()) + pg_version = v.major if v.major >= 10 else float(f"{v.major}.{v.minor}") trigger_file_setting = None - if pg_version < "12": + if pg_version < 12: trigger_file_setting = "trigger_file" - elif pg_version < "16": # PG 16 has removed `promote_trigger_file` config param. + elif pg_version < 16: # PG 16 has removed `promote_trigger_file` config param. trigger_file_setting = "promote_trigger_file" lines = [ @@ -132,7 +132,7 @@ def create_recovery_conf( if trigger_file_setting: lines.append("{} = {}".format(trigger_file_setting, adapt("trigger_file"))) - use_recovery_conf = (pg_version < "12") # no more recovery.conf in PG >= 12 + use_recovery_conf = (pg_version < 12) # no more recovery.conf in PG >= 12 if not restore_to_primary: if use_recovery_conf: lines.append("standby_mode = 'on'") @@ -145,7 +145,7 @@ def create_recovery_conf( if recovery_end_command: lines.append("recovery_end_command = {}".format(adapt(recovery_end_command))) if recovery_target_action: - if pg_version >= "9.5": + if pg_version >= 9.5: lines.append("recovery_target_action = '{}'".format(recovery_target_action)) elif recovery_target_action == "promote": pass # default action diff --git a/pyproject.toml b/pyproject.toml index 42016393..3b4a9632 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,9 @@ classifiers=[ "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ] diff --git a/test/base.py b/test/base.py index 79af8c9a..c8594f6b 100644 --- a/test/base.py +++ b/test/base.py @@ -7,11 +7,11 @@ import logging import os # pylint: disable=attribute-defined-outside-init -from distutils.version import LooseVersion from shutil import rmtree from tempfile import mkdtemp import psycopg2.extras +from packaging.version import Version from pghoard.config import find_pg_binary, set_and_check_config_defaults @@ -84,7 +84,7 @@ def config_template(self, override=None): "json_state_file_path": os.path.join(self.temp_dir, "state.json"), "pg_basebackup_path": os.path.join(bindir, "pg_basebackup"), } - if LooseVersion(ver) >= "10": + if Version(ver).major >= 10: config["backup_sites"][self.test_site]["pg_receivexlog_path"] = os.path.join(bindir, "pg_receivewal") if override: all_site_overrides = override.pop("backup_sites", None) diff --git a/test/conftest.py b/test/conftest.py index dffd51a0..d517f217 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,13 +17,13 @@ import time from contextlib import suppress from dataclasses import dataclass -from distutils.version import LooseVersion from pathlib import Path from typing import Callable, Dict, Iterator, Optional, Sequence, Union from unittest import SkipTest import psycopg2 import pytest +from packaging.version import Version from py import path as py_path # pylint: disable=no-name-in-module from rohmu.snappyfile import snappy @@ -35,7 +35,7 @@ logutil.configure_logging() -DEFAULT_PG_VERSIONS = ["16", "15", "14", "13", "12", "11", "10"] +DEFAULT_PG_VERSIONS = ["16", "15", "14", "13", "12"] def port_is_listening(hostname: str, port: int, timeout: float = 0.5) -> bool: @@ -255,7 +255,7 @@ def fixture_recovery_db(pg_version: str) -> Iterator[PGTester]: "recovery_target_timeline = 'latest'", "restore_command = 'false'", ] - if LooseVersion(pg.pgver) >= "12": + if Version(pg.pgver).major >= 12: with open(os.path.join(pg.pgdata, "standby.signal"), "w") as fp: pass diff --git a/test/test_webserver.py b/test/test_webserver.py index f68b7d2c..c52153d2 100644 --- a/test/test_webserver.py +++ b/test/test_webserver.py @@ -11,13 +11,13 @@ import threading import time from collections import deque -from distutils.version import LooseVersion from http.client import HTTPConnection from queue import Queue from unittest import mock import psycopg2 import pytest +from packaging.version import Version from rohmu.encryptor import Encryptor from pghoard import postgres_command, wal @@ -314,7 +314,7 @@ def write_dummy_wal(inc): "recovery_target_timeline = 'latest'", "restore_command = 'false'", ] - if LooseVersion(db.pgver) >= "12": + if Version(db.pgver).major >= 12: with open(os.path.join(db.pgdata, "standby.signal"), "w") as fp: pass