Merge pull request #169 from Aiven-Open/exg-ruff

Switch to ruff for formatting and linting

.github/workflows/build.yml

@@ -40,7 +40,7 @@ jobs:
           pip install --upgrade -r requirements.txt
           pip install --upgrade -r requirements.dev.txt
 
-      - id: pylint
+      - id: lint
         run: make lint
 
       - id: pip-freeze

Makefile

@@ -28,16 +28,16 @@ unittest:
 
 .PHONY: lint
 lint:
-	$(PYTHON) -m pylint --rcfile .pylintrc $(PYTHON_SOURCE_DIRS)
+	ruff check $(PYTHON_SOURCE_DIRS)
 
 .PHONY: mypy
 mypy:
 	$(PYTHON) -m mypy --show-error-codes $(PYTHON_SOURCE_DIRS)
 
 .PHONY: fmt
 fmt:
-	isort $(PYTHON_SOURCE_DIRS)
-	black $(PYTHON_SOURCE_DIRS)
+	ruff check --select I --fix $(PYTHON_SOURCE_DIRS)
+	ruff format $(PYTHON_SOURCE_DIRS)
 	$(PYTHON) -m rstfmt $(RST_FILES) -w 100
 
 .PHONY: coverage

make_release.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
-import argparse
 from pathlib import Path
+
+import argparse
 import re
 import subprocess
 
@@ -14,7 +15,8 @@ def make_release(version: str) -> None:
     subprocess.run(["git", "-C", str(project_directory), "add", str(version_filename)], check=True)
     subprocess.run(["git", "-C", str(project_directory), "commit", "-m", f"Bump to version {version}"], check=True)
     subprocess.run(
-        ["git", "-C", str(project_directory), "tag", "-s", "-a", f"releases/{version}", "-m", f"Version {version}"], check=True
+        ["git", "-C", str(project_directory), "tag", "-s", "-a", f"releases/{version}", "-m", f"Version {version}"],
+        check=True,
     )
     subprocess.run(["git", "-C", str(project_directory), "log", "-n", "1", "-p"], check=True)
     print("Run 'git push --follow-tags' to confirm the release")

pyproject.toml

@@ -2,13 +2,38 @@
 requires = ['setuptools', 'wheel', 'requests']
 build-backend = 'setuptools.build_meta'
 
-[tool.black]
+[tool.ruff]
+target-version = "py38"
 line-length = 125
 
-[tool.isort]
-no_sections = true
-force_alphabetical_sort = true
-combine_as_imports = true
-profile = "black"
-skip_gitignore = true
-line_length = 125
+[tool.ruff.lint]
+extend-select = [
+  "F",
+  "E",
+  "I",
+  "UP",
+  "BLE",
+  "C4",
+  "Q",
+  "TID",
+  "PL",
+  "RUF100", # unused-noqa
+]
+extend-ignore = [
+  "PLR0904", # too-many-public-methods
+  "PLR0912", # too-many-branches
+  "PLR0913", # too-many-arguments
+  "PLR0914", # too-many-locals
+  "PLR0915", # too-many-statements
+  "PLR0916", # too-many-boolean-expressions
+  "PLR2004", # magic-value-comparison
+  "UP006", # non-pep585-annotation
+  "UP007", # non-pep604-annotation
+]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+from-first = true
+lines-between-types = 1
+no-sections = true
+order-by-type = false

requirements.dev.txt

@@ -1,13 +1,10 @@
 # Kept in sync with the versions in CI
-black==23.7.0
+ruff==0.2.2
 mypy==1.5.1
-pylint==2.17.4
-isort==5.11.5
 # Unpinned dependencies, latest is fine
 botocore
 botocore-stubs
 mypy_boto3_s3
-pylint-quotes
 pytest
 pytest-cov
 pytest-mock

rohmu.spec

@@ -16,8 +16,6 @@ Requires:       python3-requests
 Requires:       python3-snappy
 Requires:       python3-zstandard
 BuildRequires:  python3-devel
-BuildRequires:  python3-flake8
-BuildRequires:  python3-pylint
 BuildRequires:  python3-pytest
 
 %undefine _missing_build_ids_terminate_build

rohmu/atomic_opener.py

@@ -73,15 +73,15 @@ def _atomic_opener(
     """
     parent_dir = final_path.parent
     if not parent_dir.exists():
-        raise IOError(f"Parent directory '{parent_dir}' must exist but is missing")
+        raise OSError(f"Parent directory '{parent_dir}' must exist but is missing")
     if "w" not in mode:
         raise ValueError("Write mode must be used to make actual sense")
 
     fd = os.open(str(parent_dir), os.O_TMPFILE | os.O_RDWR, 0o600)
     _fd_spy(fd)
     try:
         file_obj = os.fdopen(fd, mode=mode, encoding=encoding)
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         # when passing wrong mode, os.fdopen won't close input fd. When passing wrong encoding, it will.
         _fd_close_quietly(fd)
         raise

rohmu/common/statsd.py

@@ -71,7 +71,7 @@ def timing_manager(self, metric: str, tags: Optional[Tags] = None) -> Iterator[N
         tags = (tags or {}).copy()
         try:
             yield
-        except:  # noqa pylint: disable=broad-except,bare-except
+        except:
             tags["success"] = "0"
             self.timing(metric, time.monotonic() - start_time, tags=tags)
             raise
@@ -126,7 +126,7 @@ def _send(self, metric: str, metric_type: bytes, value: Union[int, float], tags:
                 parts.append(pattern.format(separator, tag, val).encode("utf-8"))
         elif self._message_format == MessageFormat.telegraf:
             for tag, val in send_tags.items():
-                parts.insert(1, f",{tag}={val}".encode("utf-8"))
+                parts.insert(1, f",{tag}={val}".encode())
         else:
             raise NotImplementedError("Unsupported message format")
 

rohmu/compat.py

@@ -5,8 +5,8 @@
 # but as rohmu is used outside pghoard itself better keep the imports and throw
 # a deprecation warning.
 
-from contextlib import suppress  # pylint: disable=unused-import
-from os import makedirs  # pylint: disable=unused-import
+from contextlib import suppress  # # noqa: F401
+from os import makedirs  # noqa: F401
 
 import warnings
 

rohmu/delta/common.py

@@ -96,7 +96,7 @@ def jsondict(self, **kw: Any) -> dict[str, Any]:
 
 class SizeLimitedFile:
     def __init__(self, *, path: AnyPath, file_size: int) -> None:
-        self._f = open(path, "rb")  # pylint: disable=consider-using-with
+        self._f = open(path, "rb")
         self._file_size = file_size
         self.tell = self._f.tell
 

rohmu/filewrap.py

@@ -10,8 +10,6 @@
 
 
 class FileWrap(io.BufferedIOBase):
-    # pylint: disable=unused-argument
-
     def __init__(self, next_fp: FileLike) -> None:
         super().__init__()
         self._next_fp: Optional[FileLike] = next_fp

rohmu/inotify.py

@@ -113,7 +113,7 @@ def log_event(self, ev_type: str, full_path: AnyPath) -> None:
 
         try:
             st = os.stat(full_path)
-        except:  # pylint: disable=bare-except
+        except:  # noqa: E722
             st = None
 
         self.log.debug("event: %s %s, %r", full_path, ev_type, st)

rohmu/object_storage/azure.py

@@ -5,7 +5,6 @@
 
 from __future__ import annotations
 
-# pylint: disable=import-error, no-name-in-module
 from azure.core.exceptions import HttpResponseError, ResourceExistsError
 from azure.storage.blob import BlobServiceClient, ContentSettings
 from rohmu.common.statsd import StatsdConfig
@@ -20,7 +19,7 @@
     ProgressProportionCallbackType,
     SourceStorageModelT,
 )
-from rohmu.object_storage.config import (  # pylint: disable=unused-import
+from rohmu.object_storage.config import (  # noqa: F401
     AZURE_ENDPOINT_SUFFIXES as ENDPOINT_SUFFIXES,
     AZURE_MAX_BLOCK_SIZE as MAX_BLOCK_SIZE,
     AzureObjectStorageConfig as Config,
@@ -135,7 +134,7 @@ def copy_file(
 
     def _copy_file_from_bucket(
         self,
-        source_bucket: "AzureTransfer",
+        source_bucket: AzureTransfer,
         source_key: str,
         destination_key: str,
         metadata: Optional[Metadata] = None,
@@ -256,7 +255,7 @@ def delete_key(self, key: str) -> None:
         try:
             blob_client = self.conn.get_blob_client(container=self.container_name, blob=path)
             result = blob_client.delete_blob()
-        except azure.core.exceptions.ResourceNotFoundError as ex:  # pylint: disable=no-member
+        except azure.core.exceptions.ResourceNotFoundError as ex:
             raise FileNotFoundFromStorageError(path) from ex
 
         self.notifier.object_deleted(key)
@@ -283,12 +282,11 @@ def _stream_blob(
         allows reading entire blob into memory at once or returning data from random offsets"""
         file_size = None
         start_range = byte_range[0] if byte_range else 0
-        chunk_size = self.conn._config.max_chunk_get_size  # type: ignore[attr-defined] # pylint: disable=protected-access
+        chunk_size = self.conn._config.max_chunk_get_size  # type: ignore[attr-defined]
         end_range = chunk_size - 1
         blob = self.conn.get_blob_client(self.container_name, key)
         while True:
             try:
-                # pylint: disable=protected-access
                 if byte_range:
                     length = min(byte_range[1] - start_range + 1, chunk_size)
                 else:
@@ -309,7 +307,7 @@ def _stream_blob(
                     end_range = file_size - 1
                 if progress_callback:
                     progress_callback(start_range, file_size)
-            except azure.core.exceptions.ResourceNotFoundError as ex:  # pylint: disable=no-member
+            except azure.core.exceptions.ResourceNotFoundError as ex:
                 if ex.status_code == 416:  # Empty file
                     return
                 raise FileNotFoundFromStorageError(key) from ex
@@ -328,7 +326,7 @@ def get_contents_to_fileobj(
         self.log.debug("Starting to fetch the contents of: %r", path)
         try:
             self._stream_blob(path, fileobj_to_store_to, byte_range, progress_callback)
-        except azure.core.exceptions.ResourceNotFoundError as ex:  # pylint: disable=no-member
+        except azure.core.exceptions.ResourceNotFoundError as ex:
             raise FileNotFoundFromStorageError(path) from ex
 
         if progress_callback:
@@ -340,7 +338,7 @@ def get_file_size(self, key: str) -> int:
         try:
             blob_client = self.conn.get_blob_client(self.container_name, path)
             return blob_client.get_blob_properties().size
-        except azure.core.exceptions.ResourceNotFoundError as ex:  # pylint: disable=no-member
+        except azure.core.exceptions.ResourceNotFoundError as ex:
             raise FileNotFoundFromStorageError(path) from ex
 
     def store_file_object(
@@ -351,7 +349,7 @@ def store_file_object(
         *,
         cache_control: Optional[str] = None,
         mimetype: Optional[str] = None,
-        multipart: Optional[bool] = None,  # pylint: disable=unused-argument
+        multipart: Optional[bool] = None,
         upload_progress_fn: IncrementalProgressCallbackType = None,
     ) -> None:
         if cache_control is not None:

rohmu/object_storage/config.py

@@ -23,7 +23,7 @@ def get_total_memory() -> Optional[int]:
     if platform.system() != "Linux":
         return None
 
-    with open("/proc/meminfo", "r", encoding="utf-8") as in_file:
+    with open("/proc/meminfo", encoding="utf-8") as in_file:
         for line in in_file:
             info = line.split()
             if info[0] == "MemTotal:" and info[-1] == "kB":