Remove compat layers for unsupported python versions

pghoard/basebackup.py

@@ -15,6 +15,7 @@
 import subprocess
 import time
 from concurrent.futures import ThreadPoolExecutor
+from contextlib import suppress
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty, Queue
@@ -26,7 +27,6 @@
 
 from pghoard.compressor import CompressionEvent
 from pghoard.rohmu import dates, errors, rohmufile
-from pghoard.rohmu.compat import suppress
 
 # pylint: disable=superfluous-parens
 from . import common, version, wal
@@ -36,7 +36,7 @@
     extract_pghoard_bb_v2_metadata, replication_connection_string_and_slot_using_pgpass, set_stream_nonblocking,
     set_subprocess_stdout_and_stderr_nonblocking, terminate_subprocess
 )
-from .patchedtarfile import tarfile
+import tarfile
 from .rohmu.delta.common import EMBEDDED_FILE_SIZE
 from .transfer import UploadEvent
 

pghoard/common.py

@@ -15,6 +15,7 @@
 import tarfile
 import tempfile
 import time
+from contextlib import suppress
 from dataclasses import dataclass, field
 from distutils.version import LooseVersion
 from pathlib import Path
@@ -23,7 +24,6 @@
 
 from pghoard import pgutil
 from pghoard.rohmu import IO_BLOCK_SIZE
-from pghoard.rohmu.compat import suppress
 from pghoard.rohmu.errors import Error, InvalidConfigurationError
 
 LOG = logging.getLogger("pghoard.common")

pghoard/pghoard.py

@@ -18,7 +18,7 @@
 import subprocess
 import sys
 import time
-from contextlib import closing
+from contextlib import closing, suppress
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty, Queue
@@ -39,7 +39,6 @@
 )
 from pghoard.receivexlog import PGReceiveXLog
 from pghoard.rohmu import dates, get_transfer, rohmufile
-from pghoard.rohmu.compat import suppress
 from pghoard.rohmu.dates import now as utc_now
 from pghoard.rohmu.errors import (FileNotFoundFromStorageError, InvalidConfigurationError)
 from pghoard.rohmu.inotify import InotifyWatcher

pghoard/pgutil.py

@@ -6,14 +6,7 @@
 See LICENSE for details
 """
 
-try:
-    from urllib.parse import (  # pylint: disable=no-name-in-module, import-error
-        parse_qs, urlparse
-    )
-except ImportError:
-    from urlparse import (  # pylint: disable=no-name-in-module, import-error
-        parse_qs, urlparse
-    )
+from urllib.parse import parse_qs, urlparse
 
 
 def create_connection_string(connection_info):

pghoard/restore.py

@@ -7,6 +7,7 @@
 import abc
 import argparse
 import base64
+import contextlib
 import datetime
 import enum
 import errno
@@ -32,7 +33,7 @@
 from requests import Session
 
 from pghoard.common import BaseBackupFormat, StrEnum
-from pghoard.rohmu import compat, dates, get_transfer, rohmufile
+from pghoard.rohmu import dates, get_transfer, rohmufile
 from pghoard.rohmu.errors import (Error, InvalidConfigurationError, MaybeRecoverableError)
 
 from . import common, config, logutil, version
@@ -678,7 +679,7 @@ def fetch_all(self):
         if self.errors:
             raise RestoreError("Backup download/extraction failed with {} errors".format(self.errors))
         self._create_tablespace_symlinks()
-        with compat.suppress(OSError):
+        with contextlib.suppress(OSError):
             os.rmdir(os.path.join(self.pgdata, "pgdata"))
 
     def _create_tablespace_symlinks(self):
@@ -698,7 +699,7 @@ def _create_tablespace_symlinks(self):
         # tar's limitations in exclude parameter behavior
         tsnames = [os.path.join("tablespaces", tsname) for tsname in self.tablespaces.keys()]
         for exclude in tsnames + ["tablespaces"]:
-            with compat.suppress(OSError):
+            with contextlib.suppress(OSError):
                 os.rmdir(os.path.join(self.pgdata, exclude))
 
     def _process_count(self):

pghoard/rohmu/compat.py

@@ -1,59 +1,17 @@
-# rohmu - compatibility functions to support older Python 3 versions
+# rohmu - This module used to contain compatible implementations for older
+# python version.
 #
-# Code mostly copied from Python 3.4.3 under the Python license.
-
-# suppress: call or simulate Pyhon 3.4 contextlib.suppress
-
-try:
-    from contextlib import \
-        suppress  # pylint: disable=import-error, no-name-in-module, unused-import
-except ImportError:
-
-    class suppress:
-        """Context manager to suppress specified exceptions
-
-        After the exception is suppressed, execution proceeds with the next
-        statement following the with statement.
-
-             with suppress(FileNotFoundError):
-                 os.remove(somefile)
-             # Execution still resumes here if the file was already removed
-        """
-        def __init__(self, *exceptions):
-            self._exceptions = exceptions
-
-        def __enter__(self):
-            pass
-
-        def __exit__(self, exctype, excinst, exctb):
-            # Unlike isinstance and issubclass, CPython exception handling
-            # currently only looks at the concrete type hierarchy (ignoring
-            # the instance and subclass checking hooks). While Guido considers
-            # that a bug rather than a feature, it's a fairly hard one to fix
-            # due to various internal implementation details. suppress provides
-            # the simpler issubclass based semantics, rather than trying to
-            # exactly reproduce the limitations of the CPython interpreter.
-            #
-            # See http://bugs.python.org/issue12029 for more details
-            return exctype is not None and issubclass(exctype, self._exceptions)
-
-
-# makedirs: call or simulate Pyhon 3.4.1 os.makedirs
-# Earlier Python versions raise an error if exist_ok flag is set and the
-# directory exists with different permissions.
-
-import os
-import sys
-
-if sys.version_info >= (3, 4, 1):
-    makedirs = os.makedirs
-else:
-
-    def makedirs(path, mode=0o777, exist_ok=False):
-        if not exist_ok:
-            return os.makedirs(path, mode)
-        try:
-            return os.makedirs(path, mode)
-        except FileExistsError:
-            pass
-        return None
+# Since we don't support those older versions anymore, this module could be removed,
+# but as rohmu is used outside pghoard itself better keep the imports and throw
+# a deprecation warning.
+
+import warnings
+from contextlib import suppress  # pylint: disable=unused-import
+from os import makedirs  # pylint: disable=unused-import
+
+warnings.warn(
+    "pghoard.rohmu.compat is deprecated, you should import "
+    "from the standard library directly instead",
+    DeprecationWarning,
+    stacklevel=2
+)

pghoard/rohmu/inotify.py

@@ -9,11 +9,10 @@
 import os
 import select
 import struct
+from contextlib import suppress
 from ctypes import c_char_p, c_int, c_uint32
 from threading import Thread
 
-from .compat import suppress
-
 
 class InotifyEvent(ctypes.Structure):
     _fields_ = [

pghoard/rohmu/object_storage/local.py

@@ -12,7 +12,6 @@
 import tempfile
 from io import BytesIO
 
-from ..compat import makedirs, suppress
 from ..errors import FileNotFoundFromStorageError, LocalFileIsRemoteFileError
 from .base import KEY_TYPE_OBJECT, KEY_TYPE_PREFIX, BaseTransfer, IterKeyItem
 
@@ -55,10 +54,10 @@ def delete_key(self, key):
             raise FileNotFoundFromStorageError(key)
         os.unlink(target_path)
         metadata_tmp_path = target_path + ".metadata_tmp"
-        with suppress(FileNotFoundError):
+        with contextlib.suppress(FileNotFoundError):
             os.unlink(metadata_tmp_path)
         metadata_path = target_path + ".metadata"
-        with suppress(FileNotFoundError):
+        with contextlib.suppress(FileNotFoundError):
             os.unlink(metadata_path)
 
     def delete_tree(self, key):
@@ -135,7 +134,7 @@ def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=N
             src_stat = os.stat(source_path)
         except FileNotFoundError:
             raise FileNotFoundFromStorageError(key)
-        with suppress(FileNotFoundError):
+        with contextlib.suppress(FileNotFoundError):
             dst_stat = os.stat(filepath_to_store_to)
             if dst_stat.st_dev == src_stat.st_dev and dst_stat.st_ino == src_stat.st_ino:
                 raise LocalFileIsRemoteFileError(source_path)
@@ -179,20 +178,20 @@ def _save_metadata(self, target_path, metadata):
 
     def store_file_from_memory(self, key, memstring, metadata=None, cache_control=None, mimetype=None):
         target_path = self.format_key_for_backend(key.strip("/"))
-        makedirs(os.path.dirname(target_path), exist_ok=True)
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
         with open(target_path, "wb") as fp:
             fp.write(memstring)
         self._save_metadata(target_path, metadata)
 
     def store_file_from_disk(self, key, filepath, metadata=None, multipart=None, cache_control=None, mimetype=None):
         target_path = self.format_key_for_backend(key.strip("/"))
         src_stat = os.stat(filepath)
-        with suppress(FileNotFoundError):
+        with contextlib.suppress(FileNotFoundError):
             dst_stat = os.stat(target_path)
             if dst_stat.st_dev == src_stat.st_dev and dst_stat.st_ino == src_stat.st_ino:
                 self._save_metadata(target_path, metadata)
                 raise LocalFileIsRemoteFileError(target_path)
-        makedirs(os.path.dirname(target_path), exist_ok=True)
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
         shutil.copyfile(filepath, target_path)
         self._save_metadata(target_path, metadata)
 
@@ -207,7 +206,7 @@ def store_file_object(
         upload_progress_fn=None
     ):  # pylint: disable=unused-argument
         target_path = self.format_key_for_backend(key.strip("/"))
-        makedirs(os.path.dirname(target_path), exist_ok=True)
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
         bytes_written = 0
         with open(target_path, "wb") as output_fp:
             while True:

pghoard/rohmu/object_storage/swift.py

@@ -7,10 +7,10 @@
 import logging
 import os
 import time
+from contextlib import suppress
 
 from swiftclient import client, exceptions  # pylint: disable=import-error
 
-from ..compat import suppress
 from ..dates import parse_timestamp
 from ..errors import FileNotFoundFromStorageError
 from .base import KEY_TYPE_OBJECT, KEY_TYPE_PREFIX, BaseTransfer, IterKeyItem

pghoard/rohmu/rohmufile.py

@@ -6,9 +6,9 @@
 """
 
 import time
+from contextlib import suppress
 
 from . import IO_BLOCK_SIZE
-from .compat import suppress
 from .compressor import CompressionFile, DecompressionFile, DecompressSink
 from .encryptor import DecryptorFile, DecryptSink, EncryptorFile
 from .errors import InvalidConfigurationError

pghoard/transfer.py

@@ -9,6 +9,7 @@
 import logging
 import os
 import time
+from contextlib import suppress
 from dataclasses import dataclass
 from io import BytesIO
 from pathlib import Path
@@ -21,7 +22,6 @@
 )
 from pghoard.fetcher import FileFetchManager
 from pghoard.rohmu import get_transfer
-from pghoard.rohmu.compat import suppress
 from pghoard.rohmu.errors import FileNotFoundFromStorageError
 
 _STATS_LOCK = Lock()

pghoard/webserver.py

@@ -11,7 +11,7 @@
 import time
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
 from queue import Empty, Queue
@@ -21,7 +21,6 @@
 from pghoard import wal
 from pghoard.common import (FileType, FileTypePrefixes, get_pg_wal_directory, json_encode)
 from pghoard.compressor import CompressionEvent
-from pghoard.rohmu.compat import suppress
 from pghoard.rohmu.errors import Error, FileNotFoundFromStorageError
 from pghoard.transfer import DownloadEvent, OperationEvents, TransferOperation
 from pghoard.version import __version__

test/base.py

@@ -14,7 +14,6 @@
 import psycopg2.extras
 
 from pghoard.config import find_pg_binary, set_and_check_config_defaults
-from pghoard.rohmu import compat
 
 CONSTANT_TEST_RSA_PUBLIC_KEY = """\
 -----BEGIN PUBLIC KEY-----
@@ -96,7 +95,7 @@ def config_template(self, override=None):
                     config["backup_sites"][site_name] = site_override
             config.update(override)
 
-        compat.makedirs(config["alert_file_dir"], exist_ok=True)
+        os.makedirs(config["alert_file_dir"], exist_ok=True)
         return set_and_check_config_defaults(config)
 
     def setup_method(self, method):

test/conftest.py

@@ -14,6 +14,7 @@
 import subprocess
 import tempfile
 import time
+from contextlib import suppress
 from distutils.version import LooseVersion
 from pathlib import Path
 from unittest import SkipTest
@@ -25,7 +26,6 @@
 from pghoard import config as pghconfig
 from pghoard import logutil, pgutil
 from pghoard.pghoard import PGHoard
-from pghoard.rohmu.compat import suppress
 from pghoard.rohmu.delta.common import EMBEDDED_FILE_SIZE, Progress
 from pghoard.rohmu.delta.snapshot import Snapshotter
 from pghoard.rohmu.snappyfile import snappy