From 9195ae6fc72befe0472ec30359a4503f4f992fa2 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 21:59:20 +0200 Subject: [PATCH 1/7] Update pre-commit hooks --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2ef6e93..09c42bae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude: ^vendor/|^tests/.*/fixtures/.* @@ -10,20 +10,20 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.6.4 + rev: v0.7.0 hooks: # Run the formatter. - id: ruff-format - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort files: \.py$ exclude: ^vendor/ - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 files: \.py$ From 1a3b838fe8ebbc9a9bd604cc14d5cb4643a512a9 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 22:14:18 +0200 Subject: [PATCH 2/7] Apply ruff automatic fixes --- .pre-commit-config.yaml | 3 + astacus/client.py | 5 +- astacus/common/access_log.py | 5 +- astacus/common/asyncstorage.py | 10 +-- astacus/common/cachingjsonstorage.py | 6 +- astacus/common/cassandra/client.py | 5 +- astacus/common/cassandra/config.py | 2 +- astacus/common/cassandra/schema.py | 13 ++-- astacus/common/cassandra/utils.py | 5 +- astacus/common/dependencies.py | 6 +- astacus/common/etcd.py | 6 +- astacus/common/exceptions.py | 5 +- astacus/common/ipc.py | 8 +- astacus/common/limiter.py | 5 +- astacus/common/m3placement.py | 10 +-- astacus/common/magic.py | 5 +- astacus/common/msgspec_glue.py | 5 +- astacus/common/op.py | 10 +-- astacus/common/progress.py | 12 ++- astacus/common/rohmustorage.py | 8 +- astacus/common/snapshot.py | 5 +- astacus/common/statsd.py | 9 +-- astacus/common/storage.py | 14 ++-- astacus/common/utils.py | 10 +-- astacus/config.py | 5 +- astacus/coordinator/api.py | 9 +-- astacus/coordinator/cleanup.py | 5 +- astacus/coordinator/cluster.py | 5 +- astacus/coordinator/config.py | 5 +- astacus/coordinator/coordinator.py | 5 +- astacus/coordinator/list.py | 5 +- astacus/coordinator/lockops.py | 6 +- astacus/coordinator/manifest.py | 5 +- astacus/coordinator/plugins/base.py | 69 +++++------------ .../plugins/cassandra/backup_steps.py | 2 +- .../coordinator/plugins/cassandra/model.py | 2 +- .../coordinator/plugins/cassandra/plugin.py | 2 +- .../plugins/cassandra/restore_steps.py | 2 +- .../coordinator/plugins/cassandra/utils.py | 2 +- .../plugins/clickhouse/__init__.py | 5 +- .../coordinator/plugins/clickhouse/client.py | 12 +-- .../coordinator/plugins/clickhouse/config.py | 5 +- .../plugins/clickhouse/dependencies.py | 19 ++--- .../coordinator/plugins/clickhouse/disks.py | 11 +-- .../coordinator/plugins/clickhouse/engines.py | 5 +- .../plugins/clickhouse/escaping.py | 5 +- .../plugins/clickhouse/file_metadata.py | 7 +- .../coordinator/plugins/clickhouse/macros.py | 5 +- .../plugins/clickhouse/manifest.py | 8 +- .../plugins/clickhouse/object_storage.py | 5 +- .../coordinator/plugins/clickhouse/parts.py | 12 +-- .../coordinator/plugins/clickhouse/plugin.py | 5 +- .../plugins/clickhouse/replication.py | 8 +- astacus/coordinator/plugins/clickhouse/sql.py | 5 +- .../coordinator/plugins/clickhouse/steps.py | 77 ++++++------------- astacus/coordinator/plugins/etcd.py | 6 +- astacus/coordinator/plugins/files.py | 6 +- astacus/coordinator/plugins/flink/__init__.py | 5 +- astacus/coordinator/plugins/flink/manifest.py | 5 +- astacus/coordinator/plugins/flink/plugin.py | 2 +- astacus/coordinator/plugins/flink/steps.py | 23 ++---- astacus/coordinator/plugins/m3db.py | 2 +- astacus/coordinator/plugins/zookeeper.py | 50 ++++-------- .../coordinator/plugins/zookeeper_config.py | 5 +- astacus/coordinator/state.py | 6 +- astacus/coordinator/storage_factory.py | 5 +- astacus/main.py | 5 +- astacus/manifest.py | 7 +- astacus/node/api.py | 7 +- astacus/node/cassandra.py | 9 +-- astacus/node/clear.py | 6 +- astacus/node/config.py | 5 +- astacus/node/download.py | 6 +- astacus/node/node.py | 5 +- astacus/node/snapshot.py | 8 +- astacus/node/snapshot_groups.py | 6 +- astacus/node/snapshot_op.py | 6 +- astacus/node/snapshotter.py | 6 +- astacus/node/sqlite_snapshot.py | 10 +-- astacus/node/state.py | 5 +- astacus/node/uploader.py | 6 +- astacus/server.py | 5 +- pyproject.toml | 4 +- tests/conftest.py | 5 +- tests/integration/__init__.py | 5 +- tests/integration/conftest.py | 9 +-- tests/integration/coordinator/__init__.py | 5 +- .../coordinator/plugins/__init__.py | 5 +- .../plugins/clickhouse/__init__.py | 5 +- .../plugins/clickhouse/conftest.py | 9 +-- .../plugins/clickhouse/test_client.py | 5 +- .../plugins/clickhouse/test_file_metadata.py | 5 +- .../plugins/clickhouse/test_plugin.py | 5 +- .../plugins/clickhouse/test_replication.py | 5 +- .../plugins/clickhouse/test_steps.py | 9 +-- .../plugins/clickhouse/test_zookeeper.py | 5 +- .../coordinator/plugins/flink/__init__.py | 5 +- .../coordinator/plugins/flink/test_steps.py | 5 +- tests/plugins/__init__.py | 5 +- tests/system/conftest.py | 9 +-- tests/system/test_astacus.py | 5 +- tests/system/test_config_reload.py | 5 +- tests/unit/common/cassandra/conftest.py | 5 +- tests/unit/common/cassandra/test_client.py | 5 +- tests/unit/common/cassandra/test_schema.py | 5 +- tests/unit/common/test_access_log.py | 5 +- tests/unit/common/test_limiter.py | 5 +- tests/unit/common/test_m3placement.py | 6 +- tests/unit/common/test_op.py | 6 +- tests/unit/common/test_op_stats.py | 6 +- tests/unit/common/test_progress.py | 6 +- tests/unit/common/test_statsd.py | 5 +- tests/unit/common/test_storage.py | 5 +- tests/unit/common/test_utils.py | 5 +- tests/unit/conftest.py | 5 +- tests/unit/coordinator/conftest.py | 5 +- .../coordinator/plugins/cassandra/builders.py | 5 +- .../coordinator/plugins/cassandra/conftest.py | 5 +- .../plugins/cassandra/test_backup_steps.py | 5 +- .../plugins/cassandra/test_plugin.py | 5 +- .../plugins/cassandra/test_restore_steps.py | 5 +- .../plugins/cassandra/test_utils.py | 5 +- .../plugins/clickhouse/__init__.py | 5 +- .../plugins/clickhouse/object_storage.py | 5 +- .../plugins/clickhouse/test_client.py | 5 +- .../plugins/clickhouse/test_config.py | 5 +- .../plugins/clickhouse/test_dependencies.py | 5 +- .../plugins/clickhouse/test_disks.py | 5 +- .../plugins/clickhouse/test_escaping.py | 5 +- .../plugins/clickhouse/test_macros.py | 5 +- .../plugins/clickhouse/test_manifest.py | 7 +- .../plugins/clickhouse/test_parts.py | 9 +-- .../plugins/clickhouse/test_replication.py | 5 +- .../plugins/clickhouse/test_steps.py | 71 ++++++++--------- tests/unit/coordinator/plugins/test_base.py | 10 +-- tests/unit/coordinator/plugins/test_m3db.py | 6 +- .../coordinator/plugins/test_zookeeper.py | 5 +- tests/unit/coordinator/test_backup.py | 5 +- tests/unit/coordinator/test_cleanup.py | 5 +- tests/unit/coordinator/test_list.py | 17 ++-- tests/unit/coordinator/test_lock.py | 5 +- tests/unit/coordinator/test_restore.py | 5 +- tests/unit/node/conftest.py | 5 +- tests/unit/node/test_node_cassandra.py | 5 +- tests/unit/node/test_node_download.py | 5 +- tests/unit/node/test_node_lock.py | 5 +- tests/unit/node/test_node_snapshot.py | 5 +- tests/unit/node/test_snapshot_groups.py | 6 +- tests/unit/node/test_snapshotter.py | 5 +- tests/unit/storage.py | 5 +- tests/unit/test_client.py | 5 +- tests/unit/test_config.py | 5 +- tests/unit/test_utils.py | 5 +- tests/utils.py | 5 +- 154 files changed, 457 insertions(+), 725 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09c42bae..0e637d47 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,9 @@ repos: # Ruff version. rev: v0.7.0 hooks: + # Run the linter. + - id: ruff + args: [ --fix ] # Run the formatter. - id: ruff-format diff --git a/astacus/client.py b/astacus/client.py index 43586d14..cd765c12 100644 --- a/astacus/client.py +++ b/astacus/client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Client commands for Astacus tool diff --git a/astacus/common/access_log.py b/astacus/common/access_log.py index a252de98..d1ba40d4 100644 --- a/astacus/common/access_log.py +++ b/astacus/common/access_log.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ from collections.abc import Sequence diff --git a/astacus/common/asyncstorage.py b/astacus/common/asyncstorage.py index a28063c2..21a983a6 100644 --- a/astacus/common/asyncstorage.py +++ b/astacus/common/asyncstorage.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ @@ -10,7 +8,7 @@ class AsyncHexDigestStorage: - """Subset of the HexDigestStorage API proxied async -> sync via starlette threadpool + """Subset of the HexDigestStorage API proxied async -> sync via starlette threadpool. Note that the access is not intentionally locked; therefore even synchronous API can be used in parallel (at least if it is safe to @@ -29,7 +27,7 @@ async def list_hexdigests(self) -> list[str]: class AsyncJsonStorage: - """Subset of the JsonStorage API proxied async -> sync via starlette threadpool + """Subset of the JsonStorage API proxied async -> sync via starlette threadpool. Note that the access is not intentionally locked; therefore even synchronous API can be used in parallel (at least if it is safe to diff --git a/astacus/common/cachingjsonstorage.py b/astacus/common/cachingjsonstorage.py index 02b5a709..9c864126 100644 --- a/astacus/common/cachingjsonstorage.py +++ b/astacus/common/cachingjsonstorage.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Caching layer on top of storage. diff --git a/astacus/common/cassandra/client.py b/astacus/common/cassandra/client.py index 09adaef2..184d0d89 100644 --- a/astacus/common/cassandra/client.py +++ b/astacus/common/cassandra/client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. This client code is originally from Cashew diff --git a/astacus/common/cassandra/config.py b/astacus/common/cassandra/config.py index a54778af..ac7d8ded 100644 --- a/astacus/common/cassandra/config.py +++ b/astacus/common/cassandra/config.py @@ -1,5 +1,5 @@ """Copyright (c) 2021 Aiven Ltd -See LICENSE for details +See LICENSE for details. Client configuration data model diff --git a/astacus/common/cassandra/schema.py b/astacus/common/cassandra/schema.py index 91f88784..965ea893 100644 --- a/astacus/common/cassandra/schema.py +++ b/astacus/common/cassandra/schema.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. Schema-related parts are based on basebackup_schema.py of Cashew @@ -106,10 +105,9 @@ def __lt__(self, o: CassandraNamed) -> bool: @classmethod def get_create_statement(cls, metadata: cm.Function) -> str: - """ - Based on Function.as_cql_query, but doesn't strip `frozen` from arguments as they need to be specified on restoration + """Based on Function.as_cql_query, but doesn't strip `frozen` from arguments as they need to be specified on restoration With default cassandra-driver the function fails to create with an error message like "Non-frozen UDTs are not - allowed inside collections: ..." + allowed inside collections: ...". """ sep = " " keyspace = cm.protect_name(metadata.keyspace) @@ -306,7 +304,7 @@ class CassandraSchema(AstacusModel): @classmethod def from_cassandra_session(cls, cas: CassandraSession) -> "CassandraSchema": - """Retrieve the schema using CassandraSession""" + """Retrieve the schema using CassandraSession.""" # Cassandra driver updates the schema in the background. Right after connect it's possible the schema hasn't been # retrieved yet, and it appears to be empty. This call should block until the schema has been retrieved, and raise # an exception if the nodes aren't in sync @@ -346,7 +344,6 @@ def restore_pre_data(self, cas: CassandraSession) -> None: - restore_post_data is done after the restore_pre_data, and actual snapshot restoration + moving of files has been done """ - for keyspace in self.keyspaces: # These may be version dependant, and in general should be recreated # during restore (and-or other configuration applying) diff --git a/astacus/common/cassandra/utils.py b/astacus/common/cassandra/utils.py index 321dc5a6..ae281036 100644 --- a/astacus/common/cassandra/utils.py +++ b/astacus/common/cassandra/utils.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. Miscellaneous utilities related to CassandraSession diff --git a/astacus/common/dependencies.py b/astacus/common/dependencies.py index a4531182..954fc980 100644 --- a/astacus/common/dependencies.py +++ b/astacus/common/dependencies.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. Dependency injection helper functions. """ diff --git a/astacus/common/etcd.py b/astacus/common/etcd.py index de81a878..37d1f885 100644 --- a/astacus/common/etcd.py +++ b/astacus/common/etcd.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Minimal etcdv3 client library on top of httpx diff --git a/astacus/common/exceptions.py b/astacus/common/exceptions.py index 261f919e..e4c5df35 100644 --- a/astacus/common/exceptions.py +++ b/astacus/common/exceptions.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/common/ipc.py b/astacus/common/ipc.py index 587f4393..43dd7eb1 100644 --- a/astacus/common/ipc.py +++ b/astacus/common/ipc.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .magic import DEFAULT_EMBEDDED_FILE_SIZE, StrEnum @@ -146,8 +145,7 @@ def create_snapshot_request( class SnapshotHash(msgspec.Struct, kw_only=True, frozen=True): - """ - This class represents something that is to be stored in the object storage. + """This class represents something that is to be stored in the object storage. size is provided mainly to allow for even loading of nodes in case same hexdigest is available from multiple nodes. diff --git a/astacus/common/limiter.py b/astacus/common/limiter.py index 20da87c1..487c1c66 100644 --- a/astacus/common/limiter.py +++ b/astacus/common/limiter.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from collections.abc import Awaitable, Iterable diff --git a/astacus/common/m3placement.py b/astacus/common/m3placement.py index 2e5fdb92..1f958924 100644 --- a/astacus/common/m3placement.py +++ b/astacus/common/m3placement.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. M3 placement handling code @@ -55,7 +53,7 @@ class M3PlacementNodeReplacement(AstacusModel): def rewrite_single_m3_placement_node( placement, *, src_pnode: M3PlacementNode, dst_pnode: M3PlacementNode, dst_isolation_group="" ): - """rewrite single m3 placement entry in-place in protobuf + """Rewrite single m3 placement entry in-place in protobuf. Relevant places ( see m3db src/cluster/generated/proto/placementpb/placement.proto which is @@ -85,7 +83,7 @@ def rewrite_single_m3_placement_node( def rewrite_m3_placement_bytes(value: bytes, replacements: Sequence[M3PlacementNodeReplacement]): - """rewrite whole binary m3 placement, with the set of node replacements""" + """Rewrite whole binary m3 placement, with the set of node replacements.""" placement = m3_placement_pb2.Placement() placement.ParseFromString(value) for replacement in replacements: diff --git a/astacus/common/magic.py b/astacus/common/magic.py index 5d769f55..7b8a0390 100644 --- a/astacus/common/magic.py +++ b/astacus/common/magic.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from enum import Enum diff --git a/astacus/common/msgspec_glue.py b/astacus/common/msgspec_glue.py index c571cdf3..dd7c34e2 100644 --- a/astacus/common/msgspec_glue.py +++ b/astacus/common/msgspec_glue.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ from pydantic import PydanticValueError diff --git a/astacus/common/op.py b/astacus/common/op.py index 3889196b..643a1d9e 100644 --- a/astacus/common/op.py +++ b/astacus/common/op.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Utility class for background operations. @@ -88,9 +87,8 @@ class OpState: class OpMixin: - """ - Convenience mixin which provides for both asynchronous as well as - synchronous op starting functionality, and active job querying + """Convenience mixin which provides for both asynchronous as well as + synchronous op starting functionality, and active job querying. """ state: OpState diff --git a/astacus/common/progress.py b/astacus/common/progress.py index 1866e932..aa41762f 100644 --- a/astacus/common/progress.py +++ b/astacus/common/progress.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ @@ -20,7 +18,7 @@ def increase_worth_reporting(value: int, new_value: int | None = None, *, total: int | None = None): """Make reporting sparser and sparser as values grow larger - report every 1.1**N or so - - if we know total, report every percent + - if we know total, report every percent. """ if new_value is None: new_value = value @@ -42,7 +40,7 @@ def increase_worth_reporting(value: int, new_value: int | None = None, *, total: class Progress(msgspec.Struct, kw_only=True): - """JSON-encodable progress meter of sorts""" + """JSON-encodable progress meter of sorts.""" handled: int = 0 failed: int = 0 @@ -68,7 +66,7 @@ def wrap(self, i: Sequence[T]) -> Iterator[T]: return None def start(self, n) -> None: - "Optional 'first' step, just for logic handling state (e.g. no progress object reuse desired)" + """Optional 'first' step, just for logic handling state (e.g. no progress object reuse desired).""" assert not self.total logger.info("start") self.add_total(n) diff --git a/astacus/common/rohmustorage.py b/astacus/common/rohmustorage.py index 44a0b111..557b18d0 100644 --- a/astacus/common/rohmustorage.py +++ b/astacus/common/rohmustorage.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Rohmu-specific actual object storage implementation @@ -103,7 +101,7 @@ class RohmuMetadata(RohmuModel): def rohmu_error_wrapper(fun): - """Wrap rohmu exceptions in astacus ones; to be seen what is complete set""" + """Wrap rohmu exceptions in astacus ones; to be seen what is complete set.""" def _f(*a, **kw): try: diff --git a/astacus/common/snapshot.py b/astacus/common/snapshot.py index ad34a3df..591d1256 100644 --- a/astacus/common/snapshot.py +++ b/astacus/common/snapshot.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.common.magic import DEFAULT_EMBEDDED_FILE_SIZE diff --git a/astacus/common/statsd.py b/astacus/common/statsd.py index 4ddf666f..ee303d86 100644 --- a/astacus/common/statsd.py +++ b/astacus/common/statsd.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. StatsD client @@ -61,7 +60,7 @@ def timing_manager(self, metric: str, tags: Tags | None = None): tags = (tags or {}).copy() try: yield - except: # noqa pylint: disable=broad-except,bare-except + except: tags["success"] = "0" self.timing(metric, time.monotonic() - start_time, tags=tags) raise @@ -110,7 +109,7 @@ def _send(self, metric: str, metric_type: bytes, value: int | float, tags: Tags parts.append(pattern.format(separator, tag, val).encode("utf-8")) elif self._message_format == MessageFormat.telegraf: for tag, val in send_tags.items(): - parts.insert(1, f",{tag}={val}".encode("utf-8")) + parts.insert(1, f",{tag}={val}".encode()) else: raise NotImplementedError("Unsupported message format") diff --git a/astacus/common/storage.py b/astacus/common/storage.py index 0d75ce94..be7e9edf 100644 --- a/astacus/common/storage.py +++ b/astacus/common/storage.py @@ -1,16 +1,14 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .exceptions import NotFoundException from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from rohmu.typing import FileLike -from typing import BinaryIO, Callable, ContextManager, ParamSpec, TypeAlias, TypeVar +from typing import BinaryIO, ContextManager, ParamSpec, TypeAlias, TypeVar import contextlib import io @@ -104,7 +102,7 @@ def copy(self) -> "Storage": ... def file_error_wrapper(fun: Callable[P, T]) -> Callable[P, T]: - """Wrap rohmu exceptions in astacus ones; to be seen what is complete set""" + """Wrap rohmu exceptions in astacus ones; to be seen what is complete set.""" def _f(*a: P.args, **kw: P.kwargs) -> T: try: @@ -116,7 +114,7 @@ def _f(*a: P.args, **kw: P.kwargs) -> T: class FileStorage(Storage): - """Implementation of the storage API, which just handles files - primarily useful for testing""" + """Implementation of the storage API, which just handles files - primarily useful for testing.""" def __init__(self, path: str | Path, *, hexdigest_suffix: str = ".dat", json_suffix: str = ".json") -> None: self.path = Path(path) diff --git a/astacus/common/utils.py b/astacus/common/utils.py index c21c6e66..afd9f485 100644 --- a/astacus/common/utils.py +++ b/astacus/common/utils.py @@ -1,6 +1,4 @@ -""" - -utils +"""utils. Copyright (c) 2020 Aiven Ltd See LICENSE for details @@ -71,7 +69,7 @@ def jsondict(self, **kw): def get_or_create_state(*, state: object, key: str, factory: Callable[[], Any]) -> Any: - """Get or create sub-state entry (using factory callback)""" + """Get or create sub-state entry (using factory callback).""" value = getattr(state, key, None) if value is None: value = factory() @@ -225,7 +223,7 @@ def exponential_backoff( duration: float | None = None, async_sleeper: AsyncSleeper | None = None, ) -> AnyIterable[int]: - """Exponential backoff iterator which works with both 'for' and 'async for' + """Exponential backoff iterator which works with both 'for' and 'async for'. First attempt is never delayed. The delays are only for retries. 'initial' is the first retry's delay. After that, each retry is @@ -346,7 +344,7 @@ def parallel_map_to(*, fun, iterable, result_callback, n=None) -> bool: def now(): - return datetime.datetime.now(datetime.timezone.utc) + return datetime.datetime.now(datetime.UTC) def monotonic_time(): diff --git a/astacus/config.py b/astacus/config.py index 8b341dc9..c4271ac6 100644 --- a/astacus/config.py +++ b/astacus/config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Root-level astacus configuration, which includes - global configuration diff --git a/astacus/coordinator/api.py b/astacus/coordinator/api.py index d9e56d90..611cb2b9 100644 --- a/astacus/coordinator/api.py +++ b/astacus/coordinator/api.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .cleanup import CleanupOp @@ -32,7 +31,7 @@ class OpName(StrEnum): - """(Long-running) operations defined in this API (for coordinator)""" + """(Long-running) operations defined in this API (for coordinator).""" backup = "backup" lock = "lock" @@ -53,7 +52,7 @@ async def root(): @router.post("/config/reload") async def config_reload(*, request: Request, c: Coordinator = Depends()): - """Reload astacus configuration""" + """Reload astacus configuration.""" config_path = os.environ.get("ASTACUS_CONFIG") assert config_path is not None config.set_global_config_from_path(request.app, config_path) diff --git a/astacus/coordinator/cleanup.py b/astacus/coordinator/cleanup.py index 372f12ac..d9a548f0 100644 --- a/astacus/coordinator/cleanup.py +++ b/astacus/coordinator/cleanup.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Database cleanup operation diff --git a/astacus/coordinator/cluster.py b/astacus/coordinator/cluster.py index 30531952..952f9d99 100644 --- a/astacus/coordinator/cluster.py +++ b/astacus/coordinator/cluster.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc, op, utils diff --git a/astacus/coordinator/config.py b/astacus/coordinator/config.py index cfe7a15c..d8cbd9da 100644 --- a/astacus/coordinator/config.py +++ b/astacus/coordinator/config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc diff --git a/astacus/coordinator/coordinator.py b/astacus/coordinator/coordinator.py index a99bdfb8..a5a48dcc 100644 --- a/astacus/coordinator/coordinator.py +++ b/astacus/coordinator/coordinator.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .plugins.base import CoordinatorPlugin, OperationContext, Step, StepFailedError, StepsContext diff --git a/astacus/coordinator/list.py b/astacus/coordinator/list.py index 1778a7d7..7c88bb5a 100644 --- a/astacus/coordinator/list.py +++ b/astacus/coordinator/list.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/coordinator/lockops.py b/astacus/coordinator/lockops.py index cb318c41..0c83298a 100644 --- a/astacus/coordinator/lockops.py +++ b/astacus/coordinator/lockops.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Implementation of the fictious 'lock' and 'unlock' API endpoints (we may or may not want them for debug purposes, but this is mostly all diff --git a/astacus/coordinator/manifest.py b/astacus/coordinator/manifest.py index f2f2d1a0..deb17ab2 100644 --- a/astacus/coordinator/manifest.py +++ b/astacus/coordinator/manifest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common import asyncstorage, ipc diff --git a/astacus/coordinator/plugins/base.py b/astacus/coordinator/plugins/base.py index 17102141..a2d5995f 100644 --- a/astacus/coordinator/plugins/base.py +++ b/astacus/coordinator/plugins/base.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Common base classes for the plugins @@ -16,10 +15,10 @@ from astacus.coordinator.cluster import Cluster, Result from astacus.coordinator.config import CoordinatorNode from astacus.coordinator.manifest import download_backup_manifest, download_backup_min_manifest -from collections import Counter +from collections import Counter, Counter as TCounter from collections.abc import Sequence, Set from starlette.concurrency import run_in_threadpool -from typing import Any, Counter as TCounter, Generic, TypeVar +from typing import Any, Generic, TypeVar import dataclasses import datetime @@ -103,8 +102,7 @@ def set_result(self, step_class: type[Step[T]], result: T) -> None: @dataclasses.dataclass class SnapshotStep(Step[Sequence[ipc.SnapshotResult]]): - """ - Request a snapshot of all files matching the `snapshot_root_globs`, on each nodes. + """Request a snapshot of all files matching the `snapshot_root_globs`, on each nodes. The snapshot for each file contains its path, size, modification time and hash, see `SnapshotFile` for details. @@ -126,9 +124,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[ip @dataclasses.dataclass class ListHexdigestsStep(Step[Set[str]]): - """ - Fetch the list of all files already present in object storage, identified by their hexdigest. - """ + """Fetch the list of all files already present in object storage, identified by their hexdigest.""" hexdigest_storage: AsyncHexDigestStorage @@ -138,8 +134,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Set[str]: @dataclasses.dataclass class UploadBlocksStep(Step[Sequence[ipc.SnapshotUploadResult]]): - """ - Upload to object storage all files that are not yet in that storage. + """Upload to object storage all files that are not yet in that storage. The list of files to upload comes from the snapshot taken on each node during the `SnapshotStep`, the list of files already uploaded come from the `ListHexdigestsStep`. @@ -174,8 +169,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[ip @dataclasses.dataclass class SnapshotClearStep(Step[Sequence[ipc.NodeResult]]): - """ - Request to clear the source hierarchy of the snapshotter on all nodes. + """Request to clear the source hierarchy of the snapshotter on all nodes. Depending on the request, this can clear either the main snapshotter or the delta snapshotter. """ @@ -194,8 +188,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[ip @dataclasses.dataclass class SnapshotReleaseStep(Step[Sequence[ipc.NodeResult]]): - """ - Request to release the files we don't need any more in the destination hierarchy. + """Request to release the files we don't need any more in the destination hierarchy. Allows to free some disk space before the next backup happens. """ @@ -224,8 +217,7 @@ def _hexdigests_from_hashes(self, hashes: Sequence[ipc.SnapshotHash] | None) -> @dataclasses.dataclass class UploadManifestStep(Step[None]): - """ - Store the backup manifest in the object storage. + """Store the backup manifest in the object storage. The backup manifest contains the snapshot from the `SnapshotStep` as well as the statistics collected by the `UploadBlocksStep` and the plugin manifest. @@ -265,8 +257,7 @@ def _make_backup_name(self, context: StepsContext) -> str: @dataclasses.dataclass class BackupNameStep(Step[str]): - """ - Select the name of the backup to restore. + """Select the name of the backup to restore. If the backup name was not specified in the restore request, this will select the most recent backup available in object storage, and fail if there are no backup. @@ -285,9 +276,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> str: @dataclasses.dataclass class BackupManifestStep(Step[ipc.BackupManifest]): - """ - Download the backup manifest from object storage. - """ + """Download the backup manifest from object storage.""" json_storage: AsyncJsonStorage @@ -299,9 +288,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> ipc.BackupM @dataclasses.dataclass class MapNodesStep(Step[Sequence[int | None]]): - """ - Create an index mapping nodes from cluster configuration to nodes in the backup manifest. - """ + """Create an index mapping nodes from cluster configuration to nodes in the backup manifest.""" partial_restore_nodes: Sequence[ipc.PartialRestoreRequestNode] | None = None @@ -324,9 +311,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[in @dataclasses.dataclass class RestoreStep(Step[Sequence[ipc.NodeResult]]): - """ - Request each node to download and restore all files listed in the backup manifest. - """ + """Request each node to download and restore all files listed in the backup manifest.""" storage_name: str partial_restore_nodes: Sequence[ipc.PartialRestoreRequestNode] | None = None @@ -379,9 +364,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[ip @dataclasses.dataclass class ListBackupsStep(Step[set[str]]): - """ - List all available backups and return their name. - """ + """List all available backups and return their name.""" json_storage: AsyncJsonStorage @@ -391,9 +374,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> set[str]: @dataclasses.dataclass class ListDeltaBackupsStep(Step[set[str]]): - """ - List all available delta backups and return their name. - """ + """List all available delta backups and return their name.""" json_storage: AsyncJsonStorage @@ -403,8 +384,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> set[str]: @dataclasses.dataclass class DeltaManifestsStep(Step[Sequence[ipc.BackupManifest]]): - """ - Download and parse all delta manifests necessary for restore. + """Download and parse all delta manifests necessary for restore. Includes only the deltas created after the base backup selected for restore. Returns manifests sorted by start time. @@ -430,9 +410,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[ip @dataclasses.dataclass class RestoreDeltasStep(Step[None]): - """ - Restore the delta backups: download and apply to the node. - """ + """Restore the delta backups: download and apply to the node.""" json_storage: AsyncJsonStorage storage_name: str @@ -589,8 +567,7 @@ def _prune_manifests(manifests: Sequence[ipc.ManifestMin], retention: Retention) @dataclasses.dataclass class ComputeKeptBackupsStep(Step[Sequence[ipc.ManifestMin]]): - """ - Return a list of backup manifests we want to keep, after excluding the explicitly deleted + """Return a list of backup manifests we want to keep, after excluding the explicitly deleted backups and applying the retention rules. """ @@ -632,9 +609,7 @@ async def compute_kept_deltas( @dataclasses.dataclass class DeleteBackupManifestsStep(Step[set[str]]): - """ - Delete all backup manifests that are not kept. - """ + """Delete all backup manifests that are not kept.""" json_storage: AsyncJsonStorage @@ -659,9 +634,7 @@ def get_all_backups(self, context: StepsContext) -> set[str]: @dataclasses.dataclass class DeleteDanglingHexdigestsStep(Step[None]): - """ - Delete all hexdigests that are not referenced by backup manifests. - """ + """Delete all hexdigests that are not referenced by backup manifests.""" hexdigest_storage: AsyncHexDigestStorage json_storage: AsyncJsonStorage diff --git a/astacus/coordinator/plugins/cassandra/backup_steps.py b/astacus/coordinator/plugins/cassandra/backup_steps.py index 64135a12..4566aa7c 100644 --- a/astacus/coordinator/plugins/cassandra/backup_steps.py +++ b/astacus/coordinator/plugins/cassandra/backup_steps.py @@ -1,5 +1,5 @@ """Copyright (c) 2022 Aiven Ltd -See LICENSE for details +See LICENSE for details. cassandra backup/restore plugin steps diff --git a/astacus/coordinator/plugins/cassandra/model.py b/astacus/coordinator/plugins/cassandra/model.py index 539ddf68..0a94dda2 100644 --- a/astacus/coordinator/plugins/cassandra/model.py +++ b/astacus/coordinator/plugins/cassandra/model.py @@ -1,5 +1,5 @@ """Copyright (c) 2022 Aiven Ltd -See LICENSE for details +See LICENSE for details. cassandra backup/restore plugin models diff --git a/astacus/coordinator/plugins/cassandra/plugin.py b/astacus/coordinator/plugins/cassandra/plugin.py index 6213796b..4f5c2e45 100644 --- a/astacus/coordinator/plugins/cassandra/plugin.py +++ b/astacus/coordinator/plugins/cassandra/plugin.py @@ -1,5 +1,5 @@ """Copyright (c) 2020 Aiven Ltd -See LICENSE for details +See LICENSE for details. cassandra backup/restore plugin diff --git a/astacus/coordinator/plugins/cassandra/restore_steps.py b/astacus/coordinator/plugins/cassandra/restore_steps.py index c2bb5dd1..3def4389 100644 --- a/astacus/coordinator/plugins/cassandra/restore_steps.py +++ b/astacus/coordinator/plugins/cassandra/restore_steps.py @@ -1,5 +1,5 @@ """Copyright (c) 2022 Aiven Ltd -See LICENSE for details +See LICENSE for details. cassandra backup/restore plugin steps diff --git a/astacus/coordinator/plugins/cassandra/utils.py b/astacus/coordinator/plugins/cassandra/utils.py index 7c0091ca..fafc6046 100644 --- a/astacus/coordinator/plugins/cassandra/utils.py +++ b/astacus/coordinator/plugins/cassandra/utils.py @@ -1,5 +1,5 @@ """Copyright (c) 2022 Aiven Ltd -See LICENSE for details +See LICENSE for details. cassandra backup/restore plugin utilities diff --git a/astacus/coordinator/plugins/clickhouse/__init__.py b/astacus/coordinator/plugins/clickhouse/__init__.py index 9e5816bc..a83f4251 100644 --- a/astacus/coordinator/plugins/clickhouse/__init__.py +++ b/astacus/coordinator/plugins/clickhouse/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/coordinator/plugins/clickhouse/client.py b/astacus/coordinator/plugins/clickhouse/client.py index 285eb863..0624a734 100644 --- a/astacus/coordinator/plugins/clickhouse/client.py +++ b/astacus/coordinator/plugins/clickhouse/client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.utils import build_netloc, httpx_request @@ -47,8 +46,7 @@ def __init__( class HttpClickHouseClient(ClickHouseClient): - """ - ClickHouse client using the HTTP(S) protocol, the port provided + """ClickHouse client using the HTTP(S) protocol, the port provided should match the `http_port` or `https_port` server option. The client forces all connections to the `system` database, this @@ -172,9 +170,7 @@ def escape_sql_identifier(identifier: bytes) -> str: def escape_sql_string(string: bytes) -> str: - """ - Escapes single quotes and backslashes with a backslash and wraps everything between single quotes. - """ + """Escapes single quotes and backslashes with a backslash and wraps everything between single quotes.""" return _escape_bytes(string, STRING_ESCAPE_MAP, b"'") diff --git a/astacus/coordinator/plugins/clickhouse/config.py b/astacus/coordinator/plugins/clickhouse/config.py index b3e8abf8..3881aa6d 100644 --- a/astacus/coordinator/plugins/clickhouse/config.py +++ b/astacus/coordinator/plugins/clickhouse/config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from .client import ClickHouseClient, HttpClickHouseClient diff --git a/astacus/coordinator/plugins/clickhouse/dependencies.py b/astacus/coordinator/plugins/clickhouse/dependencies.py index 79d368ce..994208bf 100644 --- a/astacus/coordinator/plugins/clickhouse/dependencies.py +++ b/astacus/coordinator/plugins/clickhouse/dependencies.py @@ -1,11 +1,10 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.manifest import AccessEntity, Table -from collections.abc import Hashable, Sequence -from typing import Callable, TypeVar +from collections.abc import Callable, Hashable, Sequence +from typing import TypeVar # noinspection PyCompatibility import graphlib @@ -22,9 +21,7 @@ def sort_topologically( get_dependencies: Callable[[Node], Sequence[NodeKey]] = lambda x: [], get_dependants: Callable[[Node], Sequence[NodeKey]] = lambda x: [], ) -> list[Node]: - """ - Sort elements topologically based on their dependencies. - """ + """Sort elements topologically based on their dependencies.""" sorter = graphlib.TopologicalSorter() # type: ignore for element in nodes: element_key = get_key(element) @@ -38,8 +35,7 @@ def sort_topologically( def tables_sorted_by_dependencies(tables: Sequence[Table]) -> Sequence[Table]: - """ - Takes a list of `(database_name: str, table: Table)` and returns a new list, + """Takes a list of `(database_name: str, table: Table)` and returns a new list, sorted in a valid order to create each table sequentially. This order is required because tables can depend on each other, for example @@ -54,8 +50,7 @@ def tables_sorted_by_dependencies(tables: Sequence[Table]) -> Sequence[Table]: def access_entities_sorted_by_dependencies(access_entities: Sequence[AccessEntity]) -> Sequence[AccessEntity]: - """ - Takes a list of `AccessEntity` and returns a new list, sorted in a valid order to + """Takes a list of `AccessEntity` and returns a new list, sorted in a valid order to create each entity sequentially. This order is required because entities can depend on each other. Most of them could diff --git a/astacus/coordinator/plugins/clickhouse/disks.py b/astacus/coordinator/plugins/clickhouse/disks.py index 0011376d..e0a890b6 100644 --- a/astacus/coordinator/plugins/clickhouse/disks.py +++ b/astacus/coordinator/plugins/clickhouse/disks.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from .config import DiskConfiguration, DiskType @@ -89,8 +88,7 @@ class Disks: ) def get_snapshot_groups(self, freeze_name: str) -> Sequence[SnapshotGroup]: - """ - Returns the glob groups inside ClickHouse data dirs where frozen table parts are stored. + """Returns the glob groups inside ClickHouse data dirs where frozen table parts are stored. For local disk, the maximum embedded file size is the default one, For remote disks the embedded file size is unlimited: we want to embed all metadata files. @@ -119,8 +117,7 @@ def _get_disk(self, path_parts: Sequence[str]) -> Disk | None: return None def parse_part_file_path(self, file_path: str) -> ParsedPath: - """ - Parse component of a file path relative to one of the ClickHouse disks. + """Parse component of a file path relative to one of the ClickHouse disks. The path can be in the normal store or in the frozen shadow store: - [disk_path]/store/123/12345678-1234-1234-1234-12345678abcd/all_1_1_0/[file.ext] diff --git a/astacus/coordinator/plugins/clickhouse/engines.py b/astacus/coordinator/plugins/clickhouse/engines.py index d47079c8..6b546a12 100644 --- a/astacus/coordinator/plugins/clickhouse/engines.py +++ b/astacus/coordinator/plugins/clickhouse/engines.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ import enum diff --git a/astacus/coordinator/plugins/clickhouse/escaping.py b/astacus/coordinator/plugins/clickhouse/escaping.py index a39d2bac..71e79abc 100644 --- a/astacus/coordinator/plugins/clickhouse/escaping.py +++ b/astacus/coordinator/plugins/clickhouse/escaping.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ import re diff --git a/astacus/coordinator/plugins/clickhouse/file_metadata.py b/astacus/coordinator/plugins/clickhouse/file_metadata.py index 184fa6ec..cb105ac1 100644 --- a/astacus/coordinator/plugins/clickhouse/file_metadata.py +++ b/astacus/coordinator/plugins/clickhouse/file_metadata.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. When using remote storage disk in ClickHouse, the actual data is in object storage but there are still metadata files in local storage. This module contains functions @@ -61,7 +60,7 @@ def from_bytes(cls, content: bytes) -> "FileMetadata": ] expected_objects_count = int(file_match.group("objects_count")) expected_objects_size = int(file_match.group("objects_size_bytes")) - total_objects_size = sum((object_metadata.size_bytes for object_metadata in objects_metadata)) + total_objects_size = sum(object_metadata.size_bytes for object_metadata in objects_metadata) if len(objects_metadata) != expected_objects_count: raise InvalidFileMetadata( f"Invalid file metadata objects count: expected {expected_objects_size}, got {len(objects_metadata)}" diff --git a/astacus/coordinator/plugins/clickhouse/macros.py b/astacus/coordinator/plugins/clickhouse/macros.py index 48460ce5..6a8314e4 100644 --- a/astacus/coordinator/plugins/clickhouse/macros.py +++ b/astacus/coordinator/plugins/clickhouse/macros.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from .client import ClickHouseClient diff --git a/astacus/coordinator/plugins/clickhouse/manifest.py b/astacus/coordinator/plugins/clickhouse/manifest.py index 9a895403..d7b8abb0 100644 --- a/astacus/coordinator/plugins/clickhouse/manifest.py +++ b/astacus/coordinator/plugins/clickhouse/manifest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.utils import AstacusModel @@ -15,8 +14,7 @@ class AccessEntity(AstacusModel): - """ - An access entity can be a user, a role, a quota, etc. + """An access entity can be a user, a role, a quota, etc. See `RetrieveAccessEntitiesStep` for more info. """ diff --git a/astacus/coordinator/plugins/clickhouse/object_storage.py b/astacus/coordinator/plugins/clickhouse/object_storage.py index 48ab7f17..bf9536e1 100644 --- a/astacus/coordinator/plugins/clickhouse/object_storage.py +++ b/astacus/coordinator/plugins/clickhouse/object_storage.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from .config import DirectCopyConfig, LocalCopyConfig diff --git a/astacus/coordinator/plugins/clickhouse/parts.py b/astacus/coordinator/plugins/clickhouse/parts.py index 229adc68..85fc57ad 100644 --- a/astacus/coordinator/plugins/clickhouse/parts.py +++ b/astacus/coordinator/plugins/clickhouse/parts.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. Algorithms to help with redistributing parts across servers for tables using the Replicated family of table engines. @@ -40,8 +39,7 @@ class Part: def get_part_servers(part_files: Iterable[PartFile]) -> Set[int]: - """ - Return the list of server indices where the part made of all these files is present. + """Return the list of server indices where the part made of all these files is present. Raises a ValueError if not all servers contain all files. """ @@ -64,9 +62,7 @@ def list_parts_to_attach( disks: Disks, tables_by_uuid: Mapping[uuid.UUID, Table], ) -> Sequence[tuple[str, bytes]]: - """ - Returns a list of table identifiers and part names to attach from the snapshot. - """ + """Returns a list of table identifiers and part names to attach from the snapshot.""" parts_to_attach: set[tuple[str, bytes]] = set() assert snapshot_result.state is not None for snapshot_file in snapshot_result.state.files: diff --git a/astacus/coordinator/plugins/clickhouse/plugin.py b/astacus/coordinator/plugins/clickhouse/plugin.py index 47689e64..49ce83d2 100644 --- a/astacus/coordinator/plugins/clickhouse/plugin.py +++ b/astacus/coordinator/plugins/clickhouse/plugin.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from .config import ( diff --git a/astacus/coordinator/plugins/clickhouse/replication.py b/astacus/coordinator/plugins/clickhouse/replication.py index 94f6557e..a3a91cef 100644 --- a/astacus/coordinator/plugins/clickhouse/replication.py +++ b/astacus/coordinator/plugins/clickhouse/replication.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from .client import ClickHouseClient, escape_sql_identifier, unescape_sql_string @@ -111,8 +110,7 @@ async def get_shard_and_replica(clickhouse_client: ClickHouseClient, database_na def get_databases_replicas( replicated_databases: Sequence[ReplicatedDatabase], servers_macros: Sequence[Macros] ) -> Mapping[bytes, Sequence[DatabaseReplica]]: - """ - Get the list of replicas for each database. + """Get the list of replicas for each database. This applies macro substitution, using the macro values of each server, to the `shard` and `replica` values of each `ReplicatedDatabase`. diff --git a/astacus/coordinator/plugins/clickhouse/sql.py b/astacus/coordinator/plugins/clickhouse/sql.py index f2881167..a5628928 100644 --- a/astacus/coordinator/plugins/clickhouse/sql.py +++ b/astacus/coordinator/plugins/clickhouse/sql.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ import dataclasses diff --git a/astacus/coordinator/plugins/clickhouse/steps.py b/astacus/coordinator/plugins/clickhouse/steps.py index a5c72a73..b344aedc 100644 --- a/astacus/coordinator/plugins/clickhouse/steps.py +++ b/astacus/coordinator/plugins/clickhouse/steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from __future__ import annotations @@ -95,9 +94,7 @@ def get_setting_repr(setting_name: str, value: _T) -> str: @dataclasses.dataclass class ValidateConfigStep(Step[None]): - """ - Validates that we have the same number of astacus node and clickhouse nodes. - """ + """Validates that we have the same number of astacus node and clickhouse nodes.""" clickhouse: ClickHouseConfiguration @@ -108,8 +105,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> None: @dataclasses.dataclass class RetrieveAccessEntitiesStep(Step[Sequence[AccessEntity]]): - """ - Backups access entities (user, roles, quotas, row_policies, settings profiles) and their grants + """Backups access entities (user, roles, quotas, row_policies, settings profiles) and their grants from ZooKeeper. This requires using the replicated storage engine for users. Inside the `access_entities_path` ZooKeeper node, there is one child znode for each type of @@ -223,8 +219,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[Ke @dataclasses.dataclass class RetrieveDatabasesAndTablesStep(Step[DatabasesAndTables]): - """ - Retrieves the list of all databases that use the replicated database engine and their tables. + """Retrieves the list of all databases that use the replicated database engine and their tables. The table names, uuids and schemas of all tables are collected. The database names, uuids, shard and replica parameters are collected. @@ -288,8 +283,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> DatabasesAn @dataclasses.dataclass class RetrieveMacrosStep(Step[Sequence[Macros]]): - """ - Retrieves the value of all macros on each server. + """Retrieves the value of all macros on each server. Returns a list of `Macros` objects, each item of the list matches one server. """ @@ -302,9 +296,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> Sequence[Ma @dataclasses.dataclass class CollectObjectStorageFilesStep(Step[list[ClickHouseObjectStorageFiles]]): - """ - Collects the list of files that are referenced by metadata files in the backup. - """ + """Collects the list of files that are referenced by metadata files in the backup.""" disks: Disks @@ -340,9 +332,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> list[ClickH class CollectTieredStorageResultsStep(Step[ipc.TieredStorageResults]): - """ - ClickHouse specific tiered storage result step. - """ + """ClickHouse specific tiered storage result step.""" async def run_step(self, cluster: Cluster, context: StepsContext) -> ipc.TieredStorageResults: object_storage_files = context.get_result(CollectObjectStorageFilesStep) @@ -357,9 +347,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> ipc.TieredS @dataclasses.dataclass class PrepareClickHouseManifestStep(Step[dict[str, Any]]): - """ - Collects access entities, databases and tables from previous steps into an uploadable manifest. - """ + """Collects access entities, databases and tables from previous steps into an uploadable manifest.""" async def run_step(self, cluster: Cluster, context: StepsContext) -> dict[str, Any]: databases, tables = context.get_result(RetrieveDatabasesAndTablesStep) @@ -379,8 +367,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> dict[str, A @dataclasses.dataclass class RemoveFrozenTablesStep(Step[None]): - """ - Removes traces of previous backups that might have failed. + """Removes traces of previous backups that might have failed. When the system unfreeze flag is enabled, clears frozen parts from all disks in a single go. """ @@ -434,8 +421,7 @@ def freeze_partitions(clickhouse_client: ClickHouseClient) -> Iterable[Awaitable @dataclasses.dataclass class FreezeTablesStep(FreezeUnfreezeTablesStepBase): - """ - Creates a frozen copy of the tables that won't change while we are uploading parts of it. + """Creates a frozen copy of the tables that won't change while we are uploading parts of it. Each table is frozen separately, one after the other. This means the complete backup of all tables will not represent a single, globally consistent, point in time. @@ -458,8 +444,7 @@ def operation(self) -> str: @dataclasses.dataclass class UnfreezeTablesStep(FreezeUnfreezeTablesStepBase): - """ - Removes the frozen parts after we're done uploading them. + """Removes the frozen parts after we're done uploading them. Frozen leftovers don't immediately harm ClickHouse or cost disk space since they are hardlinks to the parts used by the real table. However, as ClickHouse starts mutating @@ -474,8 +459,7 @@ def operation(self) -> str: @dataclasses.dataclass class MoveFrozenPartsStep(Step[None]): - """ - Renames files in the snapshot manifest to match what we will need during recover. + """Renames files in the snapshot manifest to match what we will need during recover. The freeze step creates hardlinks of the table data in the `shadow/` folder, then the snapshot steps upload these file to backup storage and remember them by their @@ -512,9 +496,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> None: @dataclasses.dataclass class ClickHouseManifestStep(Step[ClickHouseManifest]): - """ - Extracts the ClickHouse plugin manifest from the main backup manifest. - """ + """Extracts the ClickHouse plugin manifest from the main backup manifest.""" async def run_step(self, cluster: Cluster, context: StepsContext) -> ClickHouseManifest: backup_manifest = context.get_result(BackupManifestStep) @@ -556,8 +538,7 @@ def get_restore_table_query(table: Table) -> bytes: @dataclasses.dataclass class RestoreReplicatedDatabasesStep(Step[None]): - """ - Re-creates replicated databases on each client and re-create all tables in each database. + """Re-creates replicated databases on each client and re-create all tables in each database. After this step, all tables will be empty. """ @@ -685,8 +666,7 @@ def _create_dbs(client: ClickHouseClient) -> Iterator[Awaitable[None]]: @dataclasses.dataclass class ListDatabaseReplicasStep(Step[DatabasesReplicas]): - """ - For each replicated database, returns the list of replicas. + """For each replicated database, returns the list of replicas. Each replica has a `shard_name` and a `replica_name`. """ @@ -714,8 +694,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> None: @dataclasses.dataclass class RestoreAccessEntitiesStep(Step[None]): - """ - Restores access entities (user, roles, quotas, row_policies, settings profiles) and their grants + """Restores access entities (user, roles, quotas, row_policies, settings profiles) and their grants to ZooKeeper. This requires using the replicated storage engine for users. The list of access entities to restore is read from the plugin manifest, which itself was @@ -853,9 +832,7 @@ async def check_row_existence(client: ClickHouseClient) -> bool: @dataclasses.dataclass class RestoreReplicaStep(Step[None]): - """ - Restore data on all tables by using `SYSTEM RESTORE REPLICA... `. - """ + """Restore data on all tables by using `SYSTEM RESTORE REPLICA... `.""" zookeeper_client: ZooKeeperClient clients: Sequence[ClickHouseClient] @@ -875,7 +852,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> None: # this is a requirement to be allowed to run restore replica. async with self.zookeeper_client.connect() as connection: for table in replicated_tables: - await connection.delete(f"/clickhouse/tables/{str(table.uuid)}", recursive=True) + await connection.delete(f"/clickhouse/tables/{table.uuid!s}", recursive=True) def _restart_replicas(client: ClickHouseClient) -> Iterator[Awaitable[None]]: yield from ( @@ -908,8 +885,7 @@ def _restore_replicas(client: ClickHouseClient) -> Iterator[Awaitable[None]]: @dataclasses.dataclass class RestoreObjectStorageFilesStep(SyncStep[None]): - """ - If the source and target disks are not the same, restore object storage files by copying them + """If the source and target disks are not the same, restore object storage files by copying them from the source to the target disk. """ @@ -944,8 +920,7 @@ def run_sync_step(self, cluster: Cluster, context: StepsContext) -> None: @dataclasses.dataclass class AttachMergeTreePartsStep(Step[None]): - """ - Restore data to all tables by using `ALTER TABLE ... ATTACH`. + """Restore data to all tables by using `ALTER TABLE ... ATTACH`. Which part are restored to which servers depends on whether the tables uses a Replicated table engine or not, see `DistributeReplicatedPartsStep` for more @@ -983,8 +958,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> None: @dataclasses.dataclass class SyncTableReplicasStep(Step[None]): - """ - Before declaring the restoration as finished, make sure all parts of replicated tables + """Before declaring the restoration as finished, make sure all parts of replicated tables are all exchanged between all nodes. """ @@ -1013,8 +987,7 @@ def _sync_replicas(client: ClickHouseClient) -> Iterator[Awaitable[None]]: @dataclasses.dataclass class DeleteDanglingObjectStorageFilesStep(SyncStep[None]): - """ - Delete object storage files that were created before the most recent backup + """Delete object storage files that were created before the most recent backup and that are not part of any backup. """ @@ -1028,7 +1001,7 @@ def run_sync_step(self, cluster: Cluster, context: StepsContext) -> None: # If we don't have at least one backup, we don't know which files are more recent # than the latest backup, so we don't do anything. return - newest_backup_start_time = max((backup_manifest.start for backup_manifest in backup_manifests)) + newest_backup_start_time = max(backup_manifest.start for backup_manifest in backup_manifests) kept_paths: dict[str, set[str]] = {} for manifest_min in backup_manifests: @@ -1036,7 +1009,7 @@ def run_sync_step(self, cluster: Cluster, context: StepsContext) -> None: clickhouse_manifest = ClickHouseManifest.from_plugin_data(manifest_data.plugin_data) for object_storage_files in clickhouse_manifest.object_storage_files: disk_kept_paths = kept_paths.setdefault(object_storage_files.disk_name, set()) - disk_kept_paths.update((file.path for file in object_storage_files.files)) + disk_kept_paths.update(file.path for file in object_storage_files.files) for disk_name, disk_kept_paths in sorted(kept_paths.items()): disk_object_storage = self.disks.create_object_storage(disk_name=disk_name) diff --git a/astacus/coordinator/plugins/etcd.py b/astacus/coordinator/plugins/etcd.py index d09fc572..197d2d4b 100644 --- a/astacus/coordinator/plugins/etcd.py +++ b/astacus/coordinator/plugins/etcd.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. ETCD backup/restore base; while in theory someone might want to backup etcd in isolation, this isn't really the tool for it. diff --git a/astacus/coordinator/plugins/files.py b/astacus/coordinator/plugins/files.py index 0f72f753..d299df95 100644 --- a/astacus/coordinator/plugins/files.py +++ b/astacus/coordinator/plugins/files.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. File backup plugin. diff --git a/astacus/coordinator/plugins/flink/__init__.py b/astacus/coordinator/plugins/flink/__init__.py index 506823d8..c5f23c49 100644 --- a/astacus/coordinator/plugins/flink/__init__.py +++ b/astacus/coordinator/plugins/flink/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/coordinator/plugins/flink/manifest.py b/astacus/coordinator/plugins/flink/manifest.py index c61454f7..9054013b 100644 --- a/astacus/coordinator/plugins/flink/manifest.py +++ b/astacus/coordinator/plugins/flink/manifest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.utils import AstacusModel diff --git a/astacus/coordinator/plugins/flink/plugin.py b/astacus/coordinator/plugins/flink/plugin.py index 890622fe..55263278 100644 --- a/astacus/coordinator/plugins/flink/plugin.py +++ b/astacus/coordinator/plugins/flink/plugin.py @@ -1,5 +1,5 @@ """Copyright (c) 2022 Aiven Ltd -See LICENSE for details +See LICENSE for details. Flink backup/restore plugin diff --git a/astacus/coordinator/plugins/flink/steps.py b/astacus/coordinator/plugins/flink/steps.py index 03a21361..6145e93e 100644 --- a/astacus/coordinator/plugins/flink/steps.py +++ b/astacus/coordinator/plugins/flink/steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.exceptions import TransientException @@ -19,9 +18,7 @@ @dataclasses.dataclass class RetrieveDataStep(Step[Mapping[str, Any]]): - """ - Backups Flink tables from ZooKeeper. - """ + """Backups Flink tables from ZooKeeper.""" zookeeper_client: ZooKeeperClient zookeeper_paths: Sequence[str] @@ -63,9 +60,7 @@ async def _get_data_recursively( @dataclasses.dataclass class PrepareFlinkManifestStep(Step[dict[str, Any]]): - """ - Collects data from previous steps into an uploadable manifest. - """ + """Collects data from previous steps into an uploadable manifest.""" async def run_step(self, cluster: Cluster, context: StepsContext) -> dict[str, Any]: return FlinkManifest(data=context.get_result(RetrieveDataStep)).dict() @@ -73,9 +68,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> dict[str, A @dataclasses.dataclass class FlinkManifestStep(Step[FlinkManifest]): - """ - Extracts the Flink plugin manifest from the main backup manifest. - """ + """Extracts the Flink plugin manifest from the main backup manifest.""" async def run_step(self, cluster: Cluster, context: StepsContext) -> FlinkManifest: backup_manifest = context.get_result(BackupManifestStep) @@ -84,9 +77,7 @@ async def run_step(self, cluster: Cluster, context: StepsContext) -> FlinkManife @dataclasses.dataclass class RestoreDataStep(Step[None]): - """ - Restores Flint tables to ZooKeeper. - """ + """Restores Flint tables to ZooKeeper.""" zookeeper_client: ZooKeeperClient zookeeper_paths: Sequence[str] @@ -103,7 +94,7 @@ async def _restore_data( ) -> None: current_node = data.get(current_node_path) if isinstance(current_node, dict): - await connection.create(zk_path, bytes()) + await connection.create(zk_path, b"") for key in current_node.keys(): await self._restore_data(connection, f"{zk_path}/{key}", current_node, key) elif current_node: diff --git a/astacus/coordinator/plugins/m3db.py b/astacus/coordinator/plugins/m3db.py index 71ebbfb1..67975a66 100644 --- a/astacus/coordinator/plugins/m3db.py +++ b/astacus/coordinator/plugins/m3db.py @@ -1,5 +1,5 @@ """Copyright (c) 2020 Aiven Ltd -See LICENSE for details +See LICENSE for details. m3db backup/restore plugin diff --git a/astacus/coordinator/plugins/zookeeper.py b/astacus/coordinator/plugins/zookeeper.py index 21f0ceec..d335a864 100644 --- a/astacus/coordinator/plugins/zookeeper.py +++ b/astacus/coordinator/plugins/zookeeper.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.exceptions import TransientException @@ -29,21 +28,16 @@ class ZooKeeperTransaction: def create(self, path: str, value: bytes) -> None: - """ - Add a create operation to the transaction. - """ + """Add a create operation to the transaction.""" raise NotImplementedError async def commit(self) -> None: - """ - Commit the transaction. - """ + """Commit the transaction.""" raise NotImplementedError class ZooKeeperConnection: - """ - A connection to a ZooKeeper cluster. + """A connection to a ZooKeeper cluster. This is a context manager, the connection is opened when entering the context manager and closed when leaving it. @@ -57,16 +51,14 @@ async def __aexit__(self, *exc_info) -> None: raise NotImplementedError async def get(self, path: str, watch: Watcher | None = None) -> bytes: - """ - Returns the value of the node with the specified `path`. + """Returns the value of the node with the specified `path`. Raises `NoNodeError` if the node does not exist. """ raise NotImplementedError async def get_children(self, path: str, watch: Watcher | None = None) -> Sequence[str]: - """ - Returns the sorted list of all children of the given `path`. + """Returns the sorted list of all children of the given `path`. Raises `NoNodeError` if the node does not exist. """ @@ -78,16 +70,14 @@ async def get_children_with_data( get_data_fault: Callable[[], None] = lambda: None, get_children_fault: Callable[[], None] = lambda: None, ) -> dict[str, bytes]: - """ - Returns a dictionary of all children of the given `path` with their data. + """Returns a dictionary of all children of the given `path` with their data. Raises `NoNodeError` if the node does not exist. """ raise NotImplementedError async def try_create(self, path: str, value: bytes) -> bool: - """ - Creates the node with the specified `path` and `value`. + """Creates the node with the specified `path` and `value`. Auto-creates all parent nodes if they don't exist. @@ -102,8 +92,7 @@ async def try_create(self, path: str, value: bytes) -> bool: return False async def create(self, path: str, value: bytes) -> None: - """ - Creates the node with the specified `path` and `value`. + """Creates the node with the specified `path` and `value`. Auto-creates all parent nodes if they don't exist. @@ -112,8 +101,7 @@ async def create(self, path: str, value: bytes) -> None: raise NotImplementedError async def set(self, path: str, value: bytes) -> None: - """ - Set the `value` node with the specified `path`. + """Set the `value` node with the specified `path`. The node must already exist. @@ -122,8 +110,7 @@ async def set(self, path: str, value: bytes) -> None: raise NotImplementedError async def delete(self, path: str, *, recursive: bool = False) -> None: - """ - Delete the node with the specified `path` and optionally its children. + """Delete the node with the specified `path` and optionally its children. Raises `NotEmptyError` if the node has children, unless `recursive` is True. @@ -132,15 +119,11 @@ async def delete(self, path: str, *, recursive: bool = False) -> None: raise NotImplementedError async def exists(self, path: str) -> bool: - """ - Check if specified node exists. - """ + """Check if specified node exists.""" raise NotImplementedError def transaction(self) -> ZooKeeperTransaction: - """ - Begin a transaction. - """ + """Begin a transaction.""" raise NotImplementedError @@ -154,8 +137,7 @@ def get_digest_auth_data(self) -> set[tuple[str, str]]: class ZooKeeperClient: - """ - A configured client to a ZooKeeper cluster. + """A configured client to a ZooKeeper cluster. This can be safely shared between multiple threads or multiple asyncio coroutines. """ @@ -320,7 +302,7 @@ class FakeZooKeeperClient(ZooKeeperClient): def __init__(self) -> None: self._storage: dict[tuple[str, ...], bytes] = {("",): b""} self._lock = asyncio.Lock() - self.connections: list["FakeZooKeeperConnection"] = [] + self.connections: list[FakeZooKeeperConnection] = [] def connect(self) -> ZooKeeperConnection: return FakeZooKeeperConnection(self) diff --git a/astacus/coordinator/plugins/zookeeper_config.py b/astacus/coordinator/plugins/zookeeper_config.py index 949cf725..cea6e165 100644 --- a/astacus/coordinator/plugins/zookeeper_config.py +++ b/astacus/coordinator/plugins/zookeeper_config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.utils import AstacusModel, build_netloc diff --git a/astacus/coordinator/state.py b/astacus/coordinator/state.py index 3da00b34..b71080aa 100644 --- a/astacus/coordinator/state.py +++ b/astacus/coordinator/state.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. This state represents the state of the coordinator. diff --git a/astacus/coordinator/storage_factory.py b/astacus/coordinator/storage_factory.py index 29eb4caa..82d3bfef 100644 --- a/astacus/coordinator/storage_factory.py +++ b/astacus/coordinator/storage_factory.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ from astacus.common.cachingjsonstorage import CachingJsonStorage diff --git a/astacus/main.py b/astacus/main.py index 90c81f7e..cac785c5 100644 --- a/astacus/main.py +++ b/astacus/main.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Main module for astacus. diff --git a/astacus/manifest.py b/astacus/manifest.py index 00b7b199..53bce957 100644 --- a/astacus/manifest.py +++ b/astacus/manifest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. Manifest dump utility. Allows to examine the contents of the object storage without a running Astacus process. @@ -102,7 +101,7 @@ def _dump_manifest_raw(mapped_file): def _create_rohmu_storage(config_path: str, storage: str) -> RohmuStorage: - with open(config_path, "r") as config_fp: + with open(config_path) as config_fp: config_json = json.load(config_fp) if "object_storage" not in config_json: raise ValueError(f"object_storage key missing in {config_path}") diff --git a/astacus/node/api.py b/astacus/node/api.py index 97dd4464..cf55bd32 100644 --- a/astacus/node/api.py +++ b/astacus/node/api.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .clear import ClearOp @@ -30,7 +29,7 @@ class OpName(StrEnum): - """(Long-running) operations defined in this API (for node)""" + """(Long-running) operations defined in this API (for node).""" cassandra = "cassandra" clear = "clear" diff --git a/astacus/node/cassandra.py b/astacus/node/cassandra.py index 171f57de..66d5ec72 100644 --- a/astacus/node/cassandra.py +++ b/astacus/node/cassandra.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. Cassandra handling that is run on every node in the Cluster @@ -39,8 +37,7 @@ def ks_table_from_backup_path(p: Path) -> tuple[str, str]: class SimpleCassandraSubOp(NodeOp[ipc.NodeRequest, ipc.NodeResult]): - """ - Generic class to handle no arguments in + no output out case subops. + """Generic class to handle no arguments in + no output out case subops. Due to that, it does not (really) care about request, and as far as result goes it only cares about progress. diff --git a/astacus/node/clear.py b/astacus/node/clear.py index d1c942f6..11194a4d 100644 --- a/astacus/node/clear.py +++ b/astacus/node/clear.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Clearing of snapshot storage (separate from empty 'download' step for clarity) diff --git a/astacus/node/config.py b/astacus/node/config.py index e80dcd11..c2511802 100644 --- a/astacus/node/config.py +++ b/astacus/node/config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common.cassandra.config import CassandraClientConfiguration diff --git a/astacus/node/download.py b/astacus/node/download.py index bd062c09..0c303f35 100644 --- a/astacus/node/download.py +++ b/astacus/node/download.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. General restore utilities that are product independent. diff --git a/astacus/node/node.py b/astacus/node/node.py index e90d7534..557a99e2 100644 --- a/astacus/node/node.py +++ b/astacus/node/node.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/node/snapshot.py b/astacus/node/snapshot.py index 9f877e40..93c056b2 100644 --- a/astacus/node/snapshot.py +++ b/astacus/node/snapshot.py @@ -1,14 +1,12 @@ -""" - -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from abc import ABC, abstractmethod from astacus.common.ipc import SnapshotFile, SnapshotHash +from collections.abc import Iterable from pathlib import Path -from typing import Iterable import threading diff --git a/astacus/node/snapshot_groups.py b/astacus/node/snapshot_groups.py index e54ce3b5..b5e0dd68 100644 --- a/astacus/node/snapshot_groups.py +++ b/astacus/node/snapshot_groups.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. Classes for working with snapshot groups. diff --git a/astacus/node/snapshot_op.py b/astacus/node/snapshot_op.py index 0d8a6f13..0cba29ea 100644 --- a/astacus/node/snapshot_op.py +++ b/astacus/node/snapshot_op.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. General snapshot utilities that are product independent. diff --git a/astacus/node/snapshotter.py b/astacus/node/snapshotter.py index 258fa909..d644fa3d 100644 --- a/astacus/node/snapshotter.py +++ b/astacus/node/snapshotter.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/node/sqlite_snapshot.py b/astacus/node/sqlite_snapshot.py index 22330867..ca2c68b0 100644 --- a/astacus/node/sqlite_snapshot.py +++ b/astacus/node/sqlite_snapshot.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ @@ -10,10 +8,10 @@ from astacus.common.progress import Progress from astacus.node.snapshot import Snapshot from astacus.node.snapshotter import Snapshotter +from collections.abc import Iterable from contextlib import closing from functools import cached_property from pathlib import Path -from typing import Iterable import logging import os @@ -23,7 +21,7 @@ if sys.version_info >= (3, 12): from typing import override else: - from typing_extensions import override + from typing import override logger = logging.getLogger(__name__) diff --git a/astacus/node/state.py b/astacus/node/state.py index 62cc3203..57c8ebc9 100644 --- a/astacus/node/state.py +++ b/astacus/node/state.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. This state represents the state of the node. diff --git a/astacus/node/uploader.py b/astacus/node/uploader.py index 66aae389..744ec699 100644 --- a/astacus/node/uploader.py +++ b/astacus/node/uploader.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ diff --git a/astacus/server.py b/astacus/server.py index 54545e0c..9806d6ed 100644 --- a/astacus/server.py +++ b/astacus/server.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. It is responsible for setting up the FastAPI app, with the sub-routers mapped ( coordinator + node) and configured (by loading configuration diff --git a/pyproject.toml b/pyproject.toml index c6230cb1..06fe780d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,6 @@ dev = [ # Needed by pre-commit to lint and test the project "pre-commit>=3.7.0", "anyio==3.5.0", - "pylint==3.0.4", "pytest-cov==3.0.0", "pytest-mock==3.10.0", "pytest-order==1.0.0", @@ -103,6 +102,7 @@ dev = [ "pytest-watch==4.2.0", "pytest==7.2.2", "mypy==1.11.2", + "ruff>=0.7.0", # Types for things that don't seem to have them "types-botocore>=1.0.2", "types-PyYAML>=6.0.12.2", @@ -188,7 +188,7 @@ disallow_incomplete_defs = true disallow_untyped_defs = true [tool.ruff] -target-version = "py311" +target-version = "py312" line-length = 125 force-exclude = true diff --git a/tests/conftest.py b/tests/conftest.py index b23c7785..95be7e1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from _pytest.config.argparsing import Parser diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 9e5816bc..a83f4251 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e4002982..ef3832d4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.utils import build_netloc @@ -87,7 +86,7 @@ def read_logs() -> None: try: try: await asyncio.wait_for(pattern_found.wait(), timeout=timeout) - except asyncio.TimeoutError as e: + except TimeoutError as e: raise PatternNotFoundError( f"Pattern {pattern!r} not found after {timeout:.3f}s in output of {str_args}" ) from e @@ -129,7 +128,7 @@ def port_is_listening(hostname: str, port: int, timeout: float = 0.5) -> bool: connection = socket.create_connection((hostname, port), timeout) connection.close() return True - except socket.error: + except OSError: return False diff --git a/tests/integration/coordinator/__init__.py b/tests/integration/coordinator/__init__.py index 9e5816bc..a83f4251 100644 --- a/tests/integration/coordinator/__init__.py +++ b/tests/integration/coordinator/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/integration/coordinator/plugins/__init__.py b/tests/integration/coordinator/plugins/__init__.py index 9e5816bc..a83f4251 100644 --- a/tests/integration/coordinator/plugins/__init__.py +++ b/tests/integration/coordinator/plugins/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/integration/coordinator/plugins/clickhouse/__init__.py b/tests/integration/coordinator/plugins/clickhouse/__init__.py index 9e5816bc..a83f4251 100644 --- a/tests/integration/coordinator/plugins/clickhouse/__init__.py +++ b/tests/integration/coordinator/plugins/clickhouse/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/integration/coordinator/plugins/clickhouse/conftest.py b/tests/integration/coordinator/plugins/clickhouse/conftest.py index 14eec4e2..c227061d 100644 --- a/tests/integration/coordinator/plugins/clickhouse/conftest.py +++ b/tests/integration/coordinator/plugins/clickhouse/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from _pytest.fixtures import FixtureRequest @@ -405,7 +404,7 @@ def setting(name: str, value: int | float | str): return [ f""" - {str(data_dir)} + {data_dir!s} debug true @@ -447,7 +446,7 @@ def setting(name: str, value: int | float | str): {setting("enable_system_unfreeze", "true")} - {str(data_dir / "users.xml")} + {data_dir / "users.xml"!s} /clickhouse/access/ diff --git a/tests/integration/coordinator/plugins/clickhouse/test_client.py b/tests/integration/coordinator/plugins/clickhouse/test_client.py index c5702f9f..3cf2fb58 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_client.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from .conftest import ClickHouseCommand, create_clickhouse_service, get_clickhouse_client diff --git a/tests/integration/coordinator/plugins/clickhouse/test_file_metadata.py b/tests/integration/coordinator/plugins/clickhouse/test_file_metadata.py index dd81834c..06cb3f58 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_file_metadata.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_file_metadata.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.file_metadata import FileMetadata, InvalidFileMetadata, ObjectMetadata diff --git a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py index 3b9cfda2..272a272e 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from _pytest.fixtures import SubRequest diff --git a/tests/integration/coordinator/plugins/clickhouse/test_replication.py b/tests/integration/coordinator/plugins/clickhouse/test_replication.py index 5595bc06..c2e682e9 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_replication.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_replication.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.replication import get_shard_and_replica diff --git a/tests/integration/coordinator/plugins/clickhouse/test_steps.py b/tests/integration/coordinator/plugins/clickhouse/test_steps.py index 5d8525b3..a2455e2b 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_steps.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from .conftest import ClickHouseCommand, create_clickhouse_cluster, get_clickhouse_client, MinioBucket @@ -86,7 +85,7 @@ async def test_retrieve_tables(ports: Ports, clickhouse_command: ClickHouseComma ), ] zookeeper_path = ( - f"/clickhouse/tables/{str(table_uuid)}/{{my_shard}}" + f"/clickhouse/tables/{table_uuid!s}/{{my_shard}}" if clickhouse_cluster.expands_uuid_in_zookeeper_path else "/clickhouse/tables/{uuid}/{my_shard}" ) @@ -97,7 +96,7 @@ async def test_retrieve_tables(ports: Ports, clickhouse_command: ClickHouseComma engine="ReplicatedMergeTree", uuid=table_uuid, create_query=( - f"CREATE TABLE `has_tablés`.`tablé_1` UUID '{str(table_uuid)}' (`thekey` UInt32) " + f"CREATE TABLE `has_tablés`.`tablé_1` UUID '{table_uuid!s}' (`thekey` UInt32) " f"ENGINE = " f"ReplicatedMergeTree('{zookeeper_path}', '{{my_replica}}') " f"ORDER BY thekey SETTINGS index_granularity = 8192" diff --git a/tests/integration/coordinator/plugins/clickhouse/test_zookeeper.py b/tests/integration/coordinator/plugins/clickhouse/test_zookeeper.py index 33311fdf..fcf92055 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_zookeeper.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_zookeeper.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.zookeeper import ( diff --git a/tests/integration/coordinator/plugins/flink/__init__.py b/tests/integration/coordinator/plugins/flink/__init__.py index 506823d8..c5f23c49 100644 --- a/tests/integration/coordinator/plugins/flink/__init__.py +++ b/tests/integration/coordinator/plugins/flink/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/integration/coordinator/plugins/flink/test_steps.py b/tests/integration/coordinator/plugins/flink/test_steps.py index 64fd3b32..6d6d57d5 100644 --- a/tests/integration/coordinator/plugins/flink/test_steps.py +++ b/tests/integration/coordinator/plugins/flink/test_steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.base import StepsContext diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py index f53be712..1ac4e6fd 100644 --- a/tests/plugins/__init__.py +++ b/tests/plugins/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/system/conftest.py b/tests/system/conftest.py index ae77a60c..9fc356f5 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from _pytest.config import Config @@ -57,7 +56,7 @@ async def background_process(program: str | Path, *args: str | Path, **kwargs) - finally: try: await asyncio.wait_for(proc.wait(), 0.1) - except asyncio.TimeoutError: + except TimeoutError: try: proc.terminate() except: @@ -68,7 +67,7 @@ async def background_process(program: str | Path, *args: str | Path, **kwargs) - try: await asyncio.wait_for(proc.wait(), 3) break - except asyncio.TimeoutError: + except TimeoutError: try: proc.kill() except: diff --git a/tests/system/test_astacus.py b/tests/system/test_astacus.py index a5242981..a8ba1492 100644 --- a/tests/system/test_astacus.py +++ b/tests/system/test_astacus.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Basic backup-restore cycle and its variations diff --git a/tests/system/test_config_reload.py b/tests/system/test_config_reload.py index 810e5781..ce8ea4d8 100644 --- a/tests/system/test_config_reload.py +++ b/tests/system/test_config_reload.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. Hot-reloading of astacus configuration """ diff --git a/tests/unit/common/cassandra/conftest.py b/tests/unit/common/cassandra/conftest.py index 4e67cb55..52cf0c81 100644 --- a/tests/unit/common/cassandra/conftest.py +++ b/tests/unit/common/cassandra/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from tests.utils import is_cassandra_driver_importable diff --git a/tests/unit/common/cassandra/test_client.py b/tests/unit/common/cassandra/test_client.py index 081d632c..93be45d9 100644 --- a/tests/unit/common/cassandra/test_client.py +++ b/tests/unit/common/cassandra/test_client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.cassandra.config import CassandraClientConfiguration diff --git a/tests/unit/common/cassandra/test_schema.py b/tests/unit/common/cassandra/test_schema.py index 5d9c274a..ae6f7aaf 100644 --- a/tests/unit/common/cassandra/test_schema.py +++ b/tests/unit/common/cassandra/test_schema.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common.cassandra import schema diff --git a/tests/unit/common/test_access_log.py b/tests/unit/common/test_access_log.py index 3d66e7ee..7fc68262 100644 --- a/tests/unit/common/test_access_log.py +++ b/tests/unit/common/test_access_log.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ from astacus.common.access_log import AccessLogLevelFilter diff --git a/tests/unit/common/test_limiter.py b/tests/unit/common/test_limiter.py index cf5a836b..a6cc1177 100644 --- a/tests/unit/common/test_limiter.py +++ b/tests/unit/common/test_limiter.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.limiter import gather_limited, Limiter diff --git a/tests/unit/common/test_m3placement.py b/tests/unit/common/test_m3placement.py index 81a67999..f2c81ece 100644 --- a/tests/unit/common/test_m3placement.py +++ b/tests/unit/common/test_m3placement.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Some tests of the m3 placement rewriter in common diff --git a/tests/unit/common/test_op.py b/tests/unit/common/test_op.py index 44dcf370..95d49667 100644 --- a/tests/unit/common/test_op.py +++ b/tests/unit/common/test_op.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. astacus.common.op tests that do not fit elsewhere diff --git a/tests/unit/common/test_op_stats.py b/tests/unit/common/test_op_stats.py index 92554bf5..521964d6 100644 --- a/tests/unit/common/test_op_stats.py +++ b/tests/unit/common/test_op_stats.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test stats sending. diff --git a/tests/unit/common/test_progress.py b/tests/unit/common/test_progress.py index 598238b2..1541e8ae 100644 --- a/tests/unit/common/test_progress.py +++ b/tests/unit/common/test_progress.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. astacus.common.progress tests diff --git a/tests/unit/common/test_statsd.py b/tests/unit/common/test_statsd.py index 44faefaf..f4d13ccf 100644 --- a/tests/unit/common/test_statsd.py +++ b/tests/unit/common/test_statsd.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that StatsClient works as advertised. diff --git a/tests/unit/common/test_storage.py b/tests/unit/common/test_storage.py index b914473c..b4f64b31 100644 --- a/tests/unit/common/test_storage.py +++ b/tests/unit/common/test_storage.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that RohmuStorage works as advertised. diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index e2b4be3b..32e6ae70 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test astacus.common.utils diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4aa93feb..91322d24 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common.cassandra.config import CassandraClientConfiguration, SNAPSHOT_NAME diff --git a/tests/unit/coordinator/conftest.py b/tests/unit/coordinator/conftest.py index 73cc9861..ba5edd61 100644 --- a/tests/unit/coordinator/conftest.py +++ b/tests/unit/coordinator/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from .test_restore import BACKUP_MANIFEST diff --git a/tests/unit/coordinator/plugins/cassandra/builders.py b/tests/unit/coordinator/plugins/cassandra/builders.py index 6515edf9..d9a32050 100644 --- a/tests/unit/coordinator/plugins/cassandra/builders.py +++ b/tests/unit/coordinator/plugins/cassandra/builders.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.common.cassandra.schema import CassandraKeyspace diff --git a/tests/unit/coordinator/plugins/cassandra/conftest.py b/tests/unit/coordinator/plugins/cassandra/conftest.py index fa58fa3e..9612cb01 100644 --- a/tests/unit/coordinator/plugins/cassandra/conftest.py +++ b/tests/unit/coordinator/plugins/cassandra/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.base import StepsContext diff --git a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py index 5496fa56..954abdb2 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ # pylint: disable=protected-access diff --git a/tests/unit/coordinator/plugins/cassandra/test_plugin.py b/tests/unit/coordinator/plugins/cassandra/test_plugin.py index 0e73b14a..1eebb3fb 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_plugin.py +++ b/tests/unit/coordinator/plugins/cassandra/test_plugin.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc diff --git a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py index b3366499..c8403dc4 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc, utils diff --git a/tests/unit/coordinator/plugins/cassandra/test_utils.py b/tests/unit/coordinator/plugins/cassandra/test_utils.py index 2d47090c..0b13781c 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_utils.py +++ b/tests/unit/coordinator/plugins/cassandra/test_utils.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc diff --git a/tests/unit/coordinator/plugins/clickhouse/__init__.py b/tests/unit/coordinator/plugins/clickhouse/__init__.py index 9e5816bc..a83f4251 100644 --- a/tests/unit/coordinator/plugins/clickhouse/__init__.py +++ b/tests/unit/coordinator/plugins/clickhouse/__init__.py @@ -1,4 +1,3 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/unit/coordinator/plugins/clickhouse/object_storage.py b/tests/unit/coordinator/plugins/clickhouse/object_storage.py index e3c50e40..7ce5f60b 100644 --- a/tests/unit/coordinator/plugins/clickhouse/object_storage.py +++ b/tests/unit/coordinator/plugins/clickhouse/object_storage.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2024 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2024 Aiven Ltd +See LICENSE for details. """ from astacus.common.statsd import StatsClient diff --git a/tests/unit/coordinator/plugins/clickhouse/test_client.py b/tests/unit/coordinator/plugins/clickhouse/test_client.py index e97882c6..c10b7e7a 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_client.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.client import ( diff --git a/tests/unit/coordinator/plugins/clickhouse/test_config.py b/tests/unit/coordinator/plugins/clickhouse/test_config.py index 89d069c3..c834530d 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_config.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.client import HttpClickHouseClient diff --git a/tests/unit/coordinator/plugins/clickhouse/test_dependencies.py b/tests/unit/coordinator/plugins/clickhouse/test_dependencies.py index 97912d09..830be88f 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_dependencies.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_dependencies.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.dependencies import ( diff --git a/tests/unit/coordinator/plugins/clickhouse/test_disks.py b/tests/unit/coordinator/plugins/clickhouse/test_disks.py index 1a5bb292..ff559ffe 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_disks.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_disks.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.common.snapshot import SnapshotGroup diff --git a/tests/unit/coordinator/plugins/clickhouse/test_escaping.py b/tests/unit/coordinator/plugins/clickhouse/test_escaping.py index 826916fa..2b6ff211 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_escaping.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_escaping.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.escaping import escape_for_file_name, unescape_from_file_name diff --git a/tests/unit/coordinator/plugins/clickhouse/test_macros.py b/tests/unit/coordinator/plugins/clickhouse/test_macros.py index a467d4d2..5d051466 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_macros.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_macros.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.client import StubClickHouseClient diff --git a/tests/unit/coordinator/plugins/clickhouse/test_manifest.py b/tests/unit/coordinator/plugins/clickhouse/test_manifest.py index 6785d620..9ced9569 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_manifest.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_manifest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.manifest import ( @@ -156,7 +155,7 @@ def test_clickhouse_table_attributes(engine: str, is_replicated: bool, requires_ def test_table_escaped_identifier() -> None: table = Table( - database="débé".encode(), name="na`me".encode(), uuid=uuid.UUID(int=0), engine="DontCare", create_query=b"" + database="débé".encode(), name=b"na`me", uuid=uuid.UUID(int=0), engine="DontCare", create_query=b"" ) assert table.escaped_sql_identifier == "`d\\xc3\\xa9b\\xc3\\xa9`.`na\\`me`" diff --git a/tests/unit/coordinator/plugins/clickhouse/test_parts.py b/tests/unit/coordinator/plugins/clickhouse/test_parts.py index d2dba690..c091bc1a 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_parts.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_parts.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.ipc import SnapshotFile, SnapshotResult, SnapshotState @@ -22,7 +21,7 @@ def create_part_files(*, table_uuid: UUID, part_name: str, digest_seed: str) -> uuid_head = str(table_uuid)[:3] return [ SnapshotFile( - relative_path=f"store/{uuid_head}/{str(table_uuid)}/detached/{part_name}/{filename}", + relative_path=f"store/{uuid_head}/{table_uuid!s}/detached/{part_name}/{filename}", file_size=file_size, mtime_ns=0, hexdigest=hashlib.sha256(f"{digest_seed}-{part_name}-{filename}".encode()).hexdigest(), @@ -41,7 +40,7 @@ def create_part_from_part_files(snapshot_files: Sequence[SnapshotFile]) -> Part: ) -T1_UUID, T2_UUID, T3_UUID = [UUID(int=i) for i in range(1, 4)] +T1_UUID, T2_UUID, T3_UUID = (UUID(int=i) for i in range(1, 4)) TABLE_1_PART_1 = create_part_files(table_uuid=T1_UUID, part_name="all_0_0_0", digest_seed="same") TABLE_1_PART_2 = create_part_files(table_uuid=T1_UUID, part_name="all_1_1_0", digest_seed="same") TABLE_1_PART_3 = create_part_files(table_uuid=T1_UUID, part_name="all_2_2_0", digest_seed="same") diff --git a/tests/unit/coordinator/plugins/clickhouse/test_replication.py b/tests/unit/coordinator/plugins/clickhouse/test_replication.py index c62eca6c..a6f47ea9 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_replication.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_replication.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.clickhouse.macros import Macros diff --git a/tests/unit/coordinator/plugins/clickhouse/test_steps.py b/tests/unit/coordinator/plugins/clickhouse/test_steps.py index 0420e944..478a8b78 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_steps.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_steps.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common.asyncstorage import AsyncJsonStorage @@ -231,17 +230,17 @@ async def create_zookeeper_access_entities(zookeeper_client: ZooKeeperClient) -> async with zookeeper_client.connect() as connection: await asyncio.gather( connection.create("/clickhouse/access/P/a_policy", str(uuid.UUID(int=1)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=1))}", b"ATTACH ROW POLICY ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=1)!s}", b"ATTACH ROW POLICY ..."), connection.create("/clickhouse/access/Q/a_quota", str(uuid.UUID(int=2)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=2))}", b"ATTACH QUOTA ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=2)!s}", b"ATTACH QUOTA ..."), connection.create("/clickhouse/access/R/a_role", str(uuid.UUID(int=3)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=3))}", b"ATTACH ROLE ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=3)!s}", b"ATTACH ROLE ..."), connection.create("/clickhouse/access/S/a_settings_profile", str(uuid.UUID(int=4)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=4))}", b"ATTACH SETTINGS PROFILE ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=4)!s}", b"ATTACH SETTINGS PROFILE ..."), connection.create("/clickhouse/access/U/jos%C3%A9", str(uuid.UUID(int=5)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=5))}", b"ATTACH USER ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=5)!s}", b"ATTACH USER ..."), connection.create("/clickhouse/access/U/z%80enjoyer", str(uuid.UUID(int=6)).encode()), - connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=6))}", b"ATTACH USER \x80 ..."), + connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=6)!s}", b"ATTACH USER \x80 ..."), ) @@ -298,9 +297,7 @@ async def test_retrieve_keeper_map_table_data() -> None: class TrappedZooKeeperClient(FakeZooKeeperClient): - """ - A fake ZooKeeper client with a trap: it will inject a concurrent write after a few reads. - """ + """A fake ZooKeeper client with a trap: it will inject a concurrent write after a few reads.""" def __init__(self) -> None: super().__init__() @@ -667,8 +664,8 @@ async def test_parse_clickhouse_manifest() -> None: context.set_result( BackupManifestStep, BackupManifest( - start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.UTC), attempt=1, snapshot_results=[], upload_results=[], @@ -1042,9 +1039,9 @@ async def test_creating_all_access_entities_can_be_retried() -> None: async with client.connect() as connection: # Simulate a first partial restoration await connection.create("/clickhouse/access/P/a_policy", str(uuid.UUID(int=1)).encode()) - await connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=1))}", b"ATTACH ROW POLICY ...") + await connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=1)!s}", b"ATTACH ROW POLICY ...") await connection.create("/clickhouse/access/Q/a_quota", str(uuid.UUID(int=2)).encode()) - await connection.create(f"/clickhouse/access/uuid/{str(uuid.UUID(int=2))}", b"ATTACH QUOTA ...") + await connection.create(f"/clickhouse/access/uuid/{uuid.UUID(int=2)!s}", b"ATTACH QUOTA ...") await step.run_step(Cluster(nodes=[]), context) await check_restored_entities(client) @@ -1059,17 +1056,17 @@ async def check_restored_entities(client: ZooKeeperClient) -> None: assert await connection.get_children("/clickhouse/access/S") == ["a_settings_profile"] assert await connection.get_children("/clickhouse/access/U") == ["jos%C3%A9", "z%80enjoyer"] assert await connection.get("/clickhouse/access/P/a_policy") == str(uuid.UUID(int=1)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=1))}") == b"ATTACH ROW POLICY ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=1)!s}") == b"ATTACH ROW POLICY ..." assert await connection.get("/clickhouse/access/Q/a_quota") == str(uuid.UUID(int=2)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=2))}") == b"ATTACH QUOTA ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=2)!s}") == b"ATTACH QUOTA ..." assert await connection.get("/clickhouse/access/R/a_role") == str(uuid.UUID(int=3)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=3))}") == b"ATTACH ROLE ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=3)!s}") == b"ATTACH ROLE ..." assert await connection.get("/clickhouse/access/S/a_settings_profile") == str(uuid.UUID(int=4)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=4))}") == b"ATTACH SETTINGS PROFILE ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=4)!s}") == b"ATTACH SETTINGS PROFILE ..." assert await connection.get("/clickhouse/access/U/jos%C3%A9") == str(uuid.UUID(int=5)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=5))}") == b"ATTACH USER ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=5)!s}") == b"ATTACH USER ..." assert await connection.get("/clickhouse/access/U/z%80enjoyer") == str(uuid.UUID(int=6)).encode() - assert await connection.get(f"/clickhouse/access/uuid/{str(uuid.UUID(int=6))}") == b"ATTACH USER \x80 ..." + assert await connection.get(f"/clickhouse/access/uuid/{uuid.UUID(int=6)!s}") == b"ATTACH USER \x80 ..." async def test_restore_replica() -> None: @@ -1116,7 +1113,7 @@ async def test_restore_replica() -> None: async def test_restore_object_storage_files(with_stats: bool) -> None: clickhouse_source_object_storage_files = SAMPLE_MANIFEST.object_storage_files[0].files object_storage_items = [ - ObjectStorageItem(key=file.path, last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.timezone.utc)) + ObjectStorageItem(key=file.path, last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)) for file in clickhouse_source_object_storage_files ] source_object_storage = MemoryObjectStorage.from_items(object_storage_items) @@ -1296,7 +1293,7 @@ async def test_attaches_all_mergetree_parts_in_manifest() -> None: context.set_result( BackupManifestStep, BackupManifest( - start=datetime.datetime(2020, 1, 2, 3, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 3, tzinfo=datetime.UTC), attempt=1, snapshot_results=[ SnapshotResult( @@ -1387,20 +1384,20 @@ async def test_delete_object_storage_files_step(tmp_path: Path) -> None: object_storage = MemoryObjectStorage.from_items( [ ObjectStorageItem( - key="not_used/and_old", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) + key="not_used/and_old", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC) ), - ObjectStorageItem(key="abc/defghi", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)), - ObjectStorageItem(key="jkl/mnopqr", last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.timezone.utc)), - ObjectStorageItem(key="stu/vwxyza", last_modified=datetime.datetime(2020, 1, 3, tzinfo=datetime.timezone.utc)), + ObjectStorageItem(key="abc/defghi", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)), + ObjectStorageItem(key="jkl/mnopqr", last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)), + ObjectStorageItem(key="stu/vwxyza", last_modified=datetime.datetime(2020, 1, 3, tzinfo=datetime.UTC)), ObjectStorageItem( - key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.timezone.utc) + key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.UTC) ), ] ) manifests = [ BackupManifest( - start=datetime.datetime(2020, 1, 2, 10, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 2, 11, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 10, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 2, 11, tzinfo=datetime.UTC), attempt=1, snapshot_results=[], upload_results=[], @@ -1420,8 +1417,8 @@ async def test_delete_object_storage_files_step(tmp_path: Path) -> None: filename="backup-2", ), BackupManifest( - start=datetime.datetime(2020, 1, 3, 10, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 3, 11, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 3, 10, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 3, 11, tzinfo=datetime.UTC), attempt=1, snapshot_results=[], upload_results=[], @@ -1452,10 +1449,10 @@ async def test_delete_object_storage_files_step(tmp_path: Path) -> None: await step.run_step(cluster, context) assert object_storage.list_items() == [ # Only not_used/and_old was deleted - ObjectStorageItem(key="abc/defghi", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)), - ObjectStorageItem(key="jkl/mnopqr", last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.timezone.utc)), - ObjectStorageItem(key="stu/vwxyza", last_modified=datetime.datetime(2020, 1, 3, tzinfo=datetime.timezone.utc)), - ObjectStorageItem(key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.timezone.utc)), + ObjectStorageItem(key="abc/defghi", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)), + ObjectStorageItem(key="jkl/mnopqr", last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)), + ObjectStorageItem(key="stu/vwxyza", last_modified=datetime.datetime(2020, 1, 3, tzinfo=datetime.UTC)), + ObjectStorageItem(key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.UTC)), ] diff --git a/tests/unit/coordinator/plugins/test_base.py b/tests/unit/coordinator/plugins/test_base.py index 6cc434ec..9244b199 100644 --- a/tests/unit/coordinator/plugins/test_base.py +++ b/tests/unit/coordinator/plugins/test_base.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc, utils @@ -220,7 +218,7 @@ async def test_compute_kept_backups( explicit_delete=[], ) context.set_result(ListBackupsStep, set(await async_json_storage.list_jsons())) - half_a_day_after_last_backup = datetime.datetime(2020, 1, 7, 5, 0, tzinfo=datetime.timezone.utc) + half_a_day_after_last_backup = datetime.datetime(2020, 1, 7, 5, 0, tzinfo=datetime.UTC) with mock.patch.object(utils, "now", lambda: half_a_day_after_last_backup): result = await step.run_step(cluster=single_node_cluster, context=context) assert {b.filename for b in result} == kept_backups @@ -379,7 +377,7 @@ async def test_upload_manifest_step_generates_correct_backup_name( single_node_cluster: Cluster, context: StepsContext, ) -> None: - context.attempt_start = datetime.datetime(2020, 1, 7, 5, 0, tzinfo=datetime.timezone.utc) + context.attempt_start = datetime.datetime(2020, 1, 7, 5, 0, tzinfo=datetime.UTC) context.set_result(SnapshotStep, [DefaultedSnapshotResult()]) context.set_result(UploadBlocksStep, [ipc.SnapshotUploadResult()]) async_json_storage = AsyncJsonStorage(storage=MemoryJsonStorage(items={})) diff --git a/tests/unit/coordinator/plugins/test_m3db.py b/tests/unit/coordinator/plugins/test_m3db.py index d661a538..f936997e 100644 --- a/tests/unit/coordinator/plugins/test_m3db.py +++ b/tests/unit/coordinator/plugins/test_m3db.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the plugin m3 specific flow (backup + restore) works diff --git a/tests/unit/coordinator/plugins/test_zookeeper.py b/tests/unit/coordinator/plugins/test_zookeeper.py index 9f3e22a8..88c8263f 100644 --- a/tests/unit/coordinator/plugins/test_zookeeper.py +++ b/tests/unit/coordinator/plugins/test_zookeeper.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.coordinator.plugins.zookeeper import ( diff --git a/tests/unit/coordinator/test_backup.py b/tests/unit/coordinator/test_backup.py index 97d44b3e..6406238a 100644 --- a/tests/unit/coordinator/test_backup.py +++ b/tests/unit/coordinator/test_backup.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the coordinator backup endpoint works. diff --git a/tests/unit/coordinator/test_cleanup.py b/tests/unit/coordinator/test_cleanup.py index 6b0ce95a..ca421fa9 100644 --- a/tests/unit/coordinator/test_cleanup.py +++ b/tests/unit/coordinator/test_cleanup.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the cleanup endpoint behaves as advertised """ diff --git a/tests/unit/coordinator/test_list.py b/tests/unit/coordinator/test_list.py index 1497e651..17107aac 100644 --- a/tests/unit/coordinator/test_list.py +++ b/tests/unit/coordinator/test_list.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the list endpoint behaves as advertised """ @@ -114,8 +113,8 @@ def fixture_backup_manifest() -> BackupManifest: The hexdigest is faked as the table UUID's last digit and a summary of the part name. """ return BackupManifest( - start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.UTC), attempt=1, snapshot_results=[ # First node @@ -243,8 +242,8 @@ def test_api_list_deduplication(backup_manifest: BackupManifest, tmp_path: Path) backups=[ ListSingleBackup( name="1", - start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.UTC), plugin=Plugin.cassandra, attempt=1, nodes=2, @@ -321,8 +320,8 @@ def test_get_cache_entries_from_list_response() -> None: def create_backup(name: str) -> ListSingleBackup: return ListSingleBackup( name=name, - start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.timezone.utc), - end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.timezone.utc), + start=datetime.datetime(2020, 1, 2, 3, 4, 5, 678, tzinfo=datetime.UTC), + end=datetime.datetime(2020, 1, 2, 5, 6, 7, 891, tzinfo=datetime.UTC), plugin=Plugin("clickhouse"), attempt=1, nodes=2, diff --git a/tests/unit/coordinator/test_lock.py b/tests/unit/coordinator/test_lock.py index a56fbd09..0e8f9d0c 100644 --- a/tests/unit/coordinator/test_lock.py +++ b/tests/unit/coordinator/test_lock.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the coordinator lock endpoint works. diff --git a/tests/unit/coordinator/test_restore.py b/tests/unit/coordinator/test_restore.py index b2254233..f27c1eaf 100644 --- a/tests/unit/coordinator/test_restore.py +++ b/tests/unit/coordinator/test_restore.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. Test that the coordinator restore endpoint works. diff --git a/tests/unit/node/conftest.py b/tests/unit/node/conftest.py index b15eee5d..549228cd 100644 --- a/tests/unit/node/conftest.py +++ b/tests/unit/node/conftest.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common import magic diff --git a/tests/unit/node/test_node_cassandra.py b/tests/unit/node/test_node_cassandra.py index 7f088ee1..b2827264 100644 --- a/tests/unit/node/test_node_cassandra.py +++ b/tests/unit/node/test_node_cassandra.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2021 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2021 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc diff --git a/tests/unit/node/test_node_download.py b/tests/unit/node/test_node_download.py index edb86a78..90fd0b92 100644 --- a/tests/unit/node/test_node_download.py +++ b/tests/unit/node/test_node_download.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc, magic, utils diff --git a/tests/unit/node/test_node_lock.py b/tests/unit/node/test_node_lock.py index bc12d1f4..8aebe1fc 100644 --- a/tests/unit/node/test_node_lock.py +++ b/tests/unit/node/test_node_lock.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from fastapi.testclient import TestClient diff --git a/tests/unit/node/test_node_snapshot.py b/tests/unit/node/test_node_snapshot.py index bb1736f5..e3a02a1a 100644 --- a/tests/unit/node/test_node_snapshot.py +++ b/tests/unit/node/test_node_snapshot.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common import ipc, magic, utils diff --git a/tests/unit/node/test_snapshot_groups.py b/tests/unit/node/test_snapshot_groups.py index c34e8507..81477f42 100644 --- a/tests/unit/node/test_snapshot_groups.py +++ b/tests/unit/node/test_snapshot_groups.py @@ -1,7 +1,5 @@ -""" - -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ diff --git a/tests/unit/node/test_snapshotter.py b/tests/unit/node/test_snapshotter.py index c0e6c911..517e49b8 100644 --- a/tests/unit/node/test_snapshotter.py +++ b/tests/unit/node/test_snapshotter.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.common.progress import Progress diff --git a/tests/unit/storage.py b/tests/unit/storage.py index 473ef426..090147a2 100644 --- a/tests/unit/storage.py +++ b/tests/unit/storage.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.common import exceptions diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index d78684a6..40c71e15 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2023 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2023 Aiven Ltd +See LICENSE for details. """ from astacus.client import _reload_config diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 8dd520f0..8a3a9f64 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus import config diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2e4a59bd..ab8a827b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2022 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2022 Aiven Ltd +See LICENSE for details. """ from collections.abc import Sequence diff --git a/tests/utils.py b/tests/utils.py index 790b98db..89a7de84 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,5 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details +"""Copyright (c) 2020 Aiven Ltd +See LICENSE for details. """ from astacus.common.rohmustorage import RohmuCompressionType, RohmuConfig From ae94e335d9f207de0e0e058ec036c02c601f5613 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 22:33:20 +0200 Subject: [PATCH 3/7] Fix remaining ruff lint errors --- astacus/common/progress.py | 3 ++- astacus/common/storage.py | 5 +++-- astacus/common/utils.py | 14 +++++++------- astacus/coordinator/api.py | 2 +- .../plugins/clickhouse/manifest.py | 2 +- astacus/node/node.py | 3 ++- astacus/node/sqlite_snapshot.py | 8 +------- pyproject.toml | 3 +++ tests/integration/conftest.py | 3 --- .../plugins/clickhouse/conftest.py | 2 +- .../plugins/clickhouse/test_plugin.py | 4 ++-- tests/plugins/asyncio_loop.py | 2 +- tests/system/conftest.py | 2 +- tests/unit/common/cassandra/test_schema.py | 4 ++-- tests/unit/common/test_limiter.py | 4 ++-- tests/unit/common/test_op.py | 5 +++-- tests/unit/common/test_progress.py | 2 +- tests/unit/common/test_statsd.py | 6 ++++-- tests/unit/common/test_storage.py | 2 +- tests/unit/common/test_utils.py | 4 ++-- .../plugins/cassandra/test_restore_steps.py | 6 +++--- .../plugins/cassandra/test_utils.py | 2 +- .../plugins/clickhouse/test_config.py | 12 ++++++++---- .../plugins/clickhouse/test_manifest.py | 6 ++---- .../plugins/clickhouse/test_steps.py | 19 +++++-------------- tests/unit/coordinator/plugins/test_base.py | 13 ++++++------- tests/unit/coordinator/test_backup.py | 2 +- tests/unit/coordinator/test_restore.py | 4 ++-- tests/unit/node/conftest.py | 4 ++-- tests/unit/node/test_node_snapshot.py | 2 +- tests/unit/test_utils.py | 2 +- 31 files changed, 72 insertions(+), 80 deletions(-) diff --git a/astacus/common/progress.py b/astacus/common/progress.py index aa41762f..5ed56f24 100644 --- a/astacus/common/progress.py +++ b/astacus/common/progress.py @@ -110,7 +110,8 @@ def upload_failure(self, hexdigest: str) -> None: self.add_fail(info=f"upload_failure {hexdigest}") def done(self) -> None: - assert self.total is not None and self.handled <= self.total + assert self.total is not None + assert self.handled <= self.total assert not self.final self.final = True logger.info("done %r", self) diff --git a/astacus/common/storage.py b/astacus/common/storage.py index be7e9edf..4cf16b6b 100644 --- a/astacus/common/storage.py +++ b/astacus/common/storage.py @@ -6,9 +6,10 @@ from .exceptions import NotFoundException from abc import ABC, abstractmethod from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager from pathlib import Path from rohmu.typing import FileLike -from typing import BinaryIO, ContextManager, ParamSpec, TypeAlias, TypeVar +from typing import BinaryIO, ParamSpec, TypeAlias, TypeVar import contextlib import io @@ -75,7 +76,7 @@ def close(self) -> None: def delete_json(self, name: str) -> None: ... @abstractmethod - def open_json_bytes(self, name: str) -> ContextManager[mmap.mmap]: ... + def open_json_bytes(self, name: str) -> AbstractContextManager[mmap.mmap]: ... @abstractmethod def list_jsons(self) -> list[str]: ... diff --git a/astacus/common/utils.py b/astacus/common/utils.py index afd9f485..60fb56df 100644 --- a/astacus/common/utils.py +++ b/astacus/common/utils.py @@ -12,11 +12,11 @@ from abc import ABC from collections import deque from collections.abc import AsyncIterable, AsyncIterator, Callable, Hashable, Iterable, Iterator, Mapping -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager from multiprocessing.dummy import Pool # fastapi + fork = bad idea from pathlib import Path from pydantic import BaseModel -from typing import Any, ContextManager, Final, Generic, IO, Literal, overload, TextIO, TypeAlias, TypeVar +from typing import Any, Final, Generic, IO, Literal, overload, TextIO, TypeAlias, TypeVar import asyncio import contextlib @@ -294,7 +294,7 @@ def __iter__(self): class SizeLimitedFile: def __init__(self, *, path, file_size): - self._f = open(path, "rb") # pylint: disable=consider-using-with + self._f = open(path, "rb") # noqa: SIM115 self._file_size = file_size self.tell = self._f.tell @@ -356,18 +356,18 @@ def monotonic_time(): @overload -def open_path_with_atomic_rename(path: os.PathLike, *, mode: UpdateBinary) -> ContextManager[IO[bytes]]: ... +def open_path_with_atomic_rename(path: os.PathLike, *, mode: UpdateBinary) -> AbstractContextManager[IO[bytes]]: ... @overload -def open_path_with_atomic_rename(path: os.PathLike, *, mode: Write) -> ContextManager[TextIO]: ... +def open_path_with_atomic_rename(path: os.PathLike, *, mode: Write) -> AbstractContextManager[TextIO]: ... @overload -def open_path_with_atomic_rename(path: os.PathLike) -> ContextManager[IO[bytes]]: ... +def open_path_with_atomic_rename(path: os.PathLike) -> AbstractContextManager[IO[bytes]]: ... -def open_path_with_atomic_rename(path: os.PathLike, *, mode: str = "w+b") -> ContextManager[TextIO | IO[bytes]]: +def open_path_with_atomic_rename(path: os.PathLike, *, mode: str = "w+b") -> AbstractContextManager[TextIO | IO[bytes]]: return _open_path_with_atomic_rename(path, mode=mode) diff --git a/astacus/coordinator/api.py b/astacus/coordinator/api.py index 611cb2b9..365beb44 100644 --- a/astacus/coordinator/api.py +++ b/astacus/coordinator/api.py @@ -179,7 +179,7 @@ async def cleanup( def op_status(*, op_name: OpName, op_id: int, c: Coordinator = Depends()): op, op_info = c.get_op_and_op_info(op_id=op_id, op_name=op_name) result = {"state": op_info.op_status} - if isinstance(op, (BackupOp, DeltaBackupOp, RestoreOp)): + if isinstance(op, BackupOp | DeltaBackupOp | RestoreOp): result["progress"] = msgspec.to_builtins(op.progress) return result diff --git a/astacus/coordinator/plugins/clickhouse/manifest.py b/astacus/coordinator/plugins/clickhouse/manifest.py index d7b8abb0..63cfbdb2 100644 --- a/astacus/coordinator/plugins/clickhouse/manifest.py +++ b/astacus/coordinator/plugins/clickhouse/manifest.py @@ -180,7 +180,7 @@ def from_plugin_data(cls, data: Mapping[str, Any]) -> Self: def encode_manifest_data(data: Any) -> Any: if isinstance(data, dict): return {key: encode_manifest_data(value) for key, value in data.items()} - if isinstance(data, (list, tuple)): + if isinstance(data, list | tuple): return [encode_manifest_data(item) for item in data] if isinstance(data, bytes): return b64encode(data).decode() diff --git a/astacus/node/node.py b/astacus/node/node.py index 557a99e2..f14247ca 100644 --- a/astacus/node/node.py +++ b/astacus/node/node.py @@ -50,7 +50,8 @@ def still_running_callback(self) -> bool: return self._still_locked_callback() def send_result(self) -> None: - assert self.req and self.result, "subclass responsibility to set up req/result before send_result" + if not (self.req and self.result): + raise AttributeError("subclass responsibility to set up req/result before send_result") if not self.req.result_url: logger.debug("send_result omitted - no result_url") return diff --git a/astacus/node/sqlite_snapshot.py b/astacus/node/sqlite_snapshot.py index ca2c68b0..fcdb9477 100644 --- a/astacus/node/sqlite_snapshot.py +++ b/astacus/node/sqlite_snapshot.py @@ -12,17 +12,11 @@ from contextlib import closing from functools import cached_property from pathlib import Path +from typing import override import logging import os import sqlite3 -import sys - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing import override - logger = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 06fe780d..8350f533 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -292,6 +292,9 @@ ignore = [ # This configuration can't be enabled since certain operations in f-strings are forbidden. # For further details, refer to: 'ci/check_string_formats.py'. "UP032", + # non-pep695-type-alias + # Syntax not supported by mypy yet, expect using an experimental flag + "UP040" ] # A list of rule codes or prefixes for which unsafe fixes should be considered safe. extend-safe-fixes = [ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ef3832d4..e5997312 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -157,8 +157,6 @@ async def create_zookeeper(ports: Ports) -> AsyncIterator[Service]: java_path = await get_command_path("java") if java_path is None: pytest.skip("java installation not found") - # newer versions of mypy should be able to infer that this is unreachable - assert False port = ports.allocate() with tempfile.TemporaryDirectory(prefix=f"zookeeper_{port}_") as data_dir_str: data_dir = Path(data_dir_str) @@ -175,7 +173,6 @@ async def create_zookeeper(ports: Ports) -> AsyncIterator[Service]: command = get_zookeeper_command(java_path=java_path, data_dir=data_dir, port=port) if command is None: pytest.skip("zookeeper installation not found") - assert False async with contextlib.AsyncExitStack() as stack: max_attempts = 10 for attempt in range(max_attempts): diff --git a/tests/integration/coordinator/plugins/clickhouse/conftest.py b/tests/integration/coordinator/plugins/clickhouse/conftest.py index c227061d..63ad6829 100644 --- a/tests/integration/coordinator/plugins/clickhouse/conftest.py +++ b/tests/integration/coordinator/plugins/clickhouse/conftest.py @@ -204,7 +204,7 @@ async def fixture_minio_bucket(minio: MinioService) -> AsyncIterator[MinioBucket yield bucket -@pytest.fixture(scope="function", name="function_minio_bucket") +@pytest.fixture(name="function_minio_bucket") async def fixture_function_minio_bucket(minio: MinioService) -> AsyncIterator[MinioBucket]: with minio.bucket(bucket_name="function-clickhouse-bucket") as bucket: yield bucket diff --git a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py index 272a272e..915442de 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py @@ -168,7 +168,7 @@ async def fixture_restored_cluster( yield clients -@pytest.fixture(scope="function", name="function_restored_cluster") +@pytest.fixture(name="function_restored_cluster") async def fixture_function_restored_cluster( ports: Ports, clickhouse_command: ClickHouseCommand, @@ -582,7 +582,7 @@ async def test_cleanup_does_not_break_object_storage_disk_files( @pytest.mark.parametrize( - "table_name,table_engine", + ("table_name", "table_engine"), [ ("default.postgresql", TableEngine.PostgreSQL), ("default.mysql", TableEngine.MySQL), diff --git a/tests/plugins/asyncio_loop.py b/tests/plugins/asyncio_loop.py index 9b6aa34e..34411f7d 100644 --- a/tests/plugins/asyncio_loop.py +++ b/tests/plugins/asyncio_loop.py @@ -229,7 +229,7 @@ def _runtime_warning_context() -> Iterator[None]: """ with warnings.catch_warnings(record=True) as _warnings: yield - rw = ["{w.filename}:{w.lineno}:{w.message}".format(w=w) for w in _warnings if w.category == RuntimeWarning] + rw = ["{w.filename}:{w.lineno}:{w.message}".format(w=w) for w in _warnings if w.category is RuntimeWarning] if rw: raise RuntimeError("{} Runtime Warning{},\n{}".format(len(rw), "" if len(rw) == 1 else "s", "\n".join(rw))) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 9fc356f5..74c2128e 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -130,7 +130,7 @@ async def wait_url_up(url: str | URL) -> None: except httpx.NetworkError as ex: logger.debug("URL %s gave exception %r", url, ex) else: - assert False, f"url {url} still not reachable" + pytest.fail(f"url {url} still not reachable") @pytest.fixture(name="rootdir") diff --git a/tests/unit/common/cassandra/test_schema.py b/tests/unit/common/cassandra/test_schema.py index ae6f7aaf..62544d45 100644 --- a/tests/unit/common/cassandra/test_schema.py +++ b/tests/unit/common/cassandra/test_schema.py @@ -68,7 +68,7 @@ def test_schema(mocker: MockerFixture) -> None: @pytest.mark.parametrize( - "strategy_class,strategy_options,expected_dcs", + ("strategy_class", "strategy_options", "expected_dcs"), [ ("SimpleStrategy", {"replication_factor": "2"}, {}), ("NetworkTopologyStrategy", {"dc1": 2, "dc2": "3"}, {"dc1": "2", "dc2": "3"}), @@ -124,7 +124,7 @@ def test_schema_keyspace_iterate_user_types_in_restore_order() -> None: @pytest.mark.parametrize( - "definition,identifiers", + ("definition", "identifiers"), [ ("foo", ["foo"]), ("map>", ["foo"]), diff --git a/tests/unit/common/test_limiter.py b/tests/unit/common/test_limiter.py index a6cc1177..45960eb9 100644 --- a/tests/unit/common/test_limiter.py +++ b/tests/unit/common/test_limiter.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize( - "limit,expected_trace", + ("limit", "expected_trace"), [ (1, ["s1", "e1", "s2", "e2", "s3", "e3"]), (2, ["s1", "s2", "e2", "s3", "e3", "e1"]), @@ -35,7 +35,7 @@ async def add_trace(start: str, sleep: float, stop: str): @pytest.mark.parametrize( - "limit,expected_trace", + ("limit", "expected_trace"), [ (1, ["s1", "e1", "s2", "e2", "s3", "e3"]), (2, ["s1", "s2", "e2", "s3", "e3", "e1"]), diff --git a/tests/unit/common/test_op.py b/tests/unit/common/test_op.py index 95d49667..e4e3ffc8 100644 --- a/tests/unit/common/test_op.py +++ b/tests/unit/common/test_op.py @@ -15,7 +15,7 @@ @pytest.mark.parametrize( - "fun_ex,expect_status,expect_ex", + ("fun_ex", "expect_status", "expect_ex"), [ # Non-failing operation winds up in None (None, op.Op.Status.done, None), @@ -52,5 +52,6 @@ async def _async(): mixin.start_op(op=op_obj, op_name="dummy", fun=_sync) await mixin.background_tasks() except Exception as ex: # pylint: disable=broad-except - assert expect_ex and isinstance(ex, expect_ex) + assert expect_ex + assert isinstance(ex, expect_ex) assert op_obj.info.op_status == expect_status diff --git a/tests/unit/common/test_progress.py b/tests/unit/common/test_progress.py index 1541e8ae..ce9fa8c6 100644 --- a/tests/unit/common/test_progress.py +++ b/tests/unit/common/test_progress.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize( - "old_value,new_value,total,exp", + ("old_value", "new_value", "total", "exp"), [ (0, 1, None, True), (8, 9, None, True), diff --git a/tests/unit/common/test_statsd.py b/tests/unit/common/test_statsd.py index f4d13ccf..64d6e851 100644 --- a/tests/unit/common/test_statsd.py +++ b/tests/unit/common/test_statsd.py @@ -48,9 +48,11 @@ async def test_statsd() -> None: with c.timing_manager("sync_timing"): pass data = await received.get() - assert data.startswith(b"sync_timing,success=1:") and data.endswith(b"|ms") + assert data.startswith(b"sync_timing,success=1:") + assert data.endswith(b"|ms") async with c.async_timing_manager("async_timing"): pass data = await received.get() - assert data.startswith(b"async_timing,success=1:") and data.endswith(b"|ms") + assert data.startswith(b"async_timing,success=1:") + assert data.endswith(b"|ms") diff --git a/tests/unit/common/test_storage.py b/tests/unit/common/test_storage.py index b4f64b31..c878c250 100644 --- a/tests/unit/common/test_storage.py +++ b/tests/unit/common/test_storage.py @@ -72,7 +72,7 @@ def _test_jsonstorage(storage: JsonStorage) -> None: @pytest.mark.parametrize( - "engine,kw,ex", + ("engine", "kw", "ex"), [ ("file", {}, None), ("rohmu", {}, None), diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 32e6ae70..78ff8e41 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -105,7 +105,7 @@ def _assert_rounded_waits_equals(x): @pytest.mark.parametrize( - "v,s", + ("v", "s"), [ (timedelta(days=1, seconds=1), "1d 1s"), (timedelta(hours=3, minutes=2, seconds=1), "3h 2m 1s"), @@ -117,7 +117,7 @@ def test_timedelta_as_short_str(v: timedelta, s: str) -> None: @pytest.mark.parametrize( - "v,s", + ("v", "s"), [ (1_234_567_901_234_678, "1234.6 TB"), (1_234_567_901_234, "1.2 TB"), diff --git a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py index c8403dc4..267dbcb4 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py @@ -59,7 +59,7 @@ async def test_step_start_cassandra(mocker: MockerFixture, override_tokens: bool ) backup_manifest = ipc.BackupManifest( - start=datetime.datetime.now(), + start=datetime.datetime.now(tz=datetime.UTC), attempt=1, snapshot_results=[ipc.SnapshotResult()], upload_results=[], @@ -156,7 +156,7 @@ async def __anext__(self): return self.data[self.index] -@pytest.mark.parametrize("steps,success", [([True], True), ([False, True], True), ([False], False)]) +@pytest.mark.parametrize(("steps", "success"), [([True], True), ([False, True], True), ([False], False)]) async def test_step_wait_cassandra_up(mocker: MockerFixture, steps: list[bool], success: bool) -> None: get_schema_steps = steps[:] @@ -180,7 +180,7 @@ async def get_schema_hash(cluster, nodes): @pytest.mark.parametrize( - "replaced_node_step, expected_nodes", [(restore_steps.StopReplacedNodesStep, [_coordinator_node(2)]), (None, None)] + ("replaced_node_step", "expected_nodes"), [(restore_steps.StopReplacedNodesStep, [_coordinator_node(2)]), (None, None)] ) async def test_stopped_nodes_for_wait_cassandra_up_step( replaced_node_step: type[restore_steps.StopReplacedNodesStep] | None, diff --git a/tests/unit/coordinator/plugins/cassandra/test_utils.py b/tests/unit/coordinator/plugins/cassandra/test_utils.py index 0b13781c..6c49c829 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_utils.py +++ b/tests/unit/coordinator/plugins/cassandra/test_utils.py @@ -33,7 +33,7 @@ async def wait_successful_results(*, start_results, result_class): @pytest.mark.parametrize( - "hashes,result", + ("hashes", "result"), [ ([], ("", "Unable to retrieve schema hash at all")), ([1, 2], ("", "Multiple schema hashes present: [1, 2]")), diff --git a/tests/unit/coordinator/plugins/clickhouse/test_config.py b/tests/unit/coordinator/plugins/clickhouse/test_config.py index c834530d..6139d9f6 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_config.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_config.py @@ -35,11 +35,15 @@ def test_get_authenticated_zookeeper_client() -> None: user=ZooKeeperConfigurationUser(username="local-user", password=SecretStr("secret")), ) client = get_zookeeper_client(configuration) - assert client is not None and isinstance(client, KazooZooKeeperClient) - assert client.user is not None and isinstance(client.user, ZooKeeperUser) + assert client is not None + assert isinstance(client, KazooZooKeeperClient) + assert client.user is not None + assert isinstance(client.user, ZooKeeperUser) connection = client.connect() - assert connection is not None and isinstance(connection, KazooZooKeeperConnection) - assert connection.client is not None and isinstance(connection.client, KazooClient) + assert connection is not None + assert isinstance(connection, KazooZooKeeperConnection) + assert connection.client is not None + assert isinstance(connection.client, KazooClient) assert connection.client.auth_data == {("digest", "local-user:secret")} diff --git a/tests/unit/coordinator/plugins/clickhouse/test_manifest.py b/tests/unit/coordinator/plugins/clickhouse/test_manifest.py index 9ced9569..b733b64c 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_manifest.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_manifest.py @@ -100,7 +100,7 @@ @pytest.mark.parametrize( - "engine,is_replicated,requires_freezing", + ("engine", "is_replicated", "requires_freezing"), [ ("AggregatingMergeTree", False, True), ("Buffer", False, False), @@ -154,9 +154,7 @@ def test_clickhouse_table_attributes(engine: str, is_replicated: bool, requires_ def test_table_escaped_identifier() -> None: - table = Table( - database="débé".encode(), name=b"na`me", uuid=uuid.UUID(int=0), engine="DontCare", create_query=b"" - ) + table = Table(database="débé".encode(), name=b"na`me", uuid=uuid.UUID(int=0), engine="DontCare", create_query=b"") assert table.escaped_sql_identifier == "`d\\xc3\\xa9b\\xc3\\xa9`.`na\\`me`" diff --git a/tests/unit/coordinator/plugins/clickhouse/test_steps.py b/tests/unit/coordinator/plugins/clickhouse/test_steps.py index 478a8b78..1defe3a2 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_steps.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_steps.py @@ -92,7 +92,6 @@ import datetime import msgspec import pytest -import sys import tempfile import uuid @@ -193,15 +192,11 @@ def mock_clickhouse_client() -> mock.Mock: mock_client = mock.Mock(spec_set=ClickHouseClient) - if sys.version_info < (3, 8): - awaitable = asyncio.Future() - awaitable.set_result(mock.Mock(spec_set=list)) - mock_client.execute.return_value = awaitable return mock_client @pytest.mark.parametrize( - "clickhouse_count,coordinator_count,success", + ("clickhouse_count", "coordinator_count", "success"), [ (3, 3, True), (0, 0, True), @@ -1383,15 +1378,11 @@ def check_each_pair_of_calls_has_the_same_session_id(mock_calls: Sequence[MockCa async def test_delete_object_storage_files_step(tmp_path: Path) -> None: object_storage = MemoryObjectStorage.from_items( [ - ObjectStorageItem( - key="not_used/and_old", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC) - ), + ObjectStorageItem(key="not_used/and_old", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)), ObjectStorageItem(key="abc/defghi", last_modified=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)), ObjectStorageItem(key="jkl/mnopqr", last_modified=datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)), ObjectStorageItem(key="stu/vwxyza", last_modified=datetime.datetime(2020, 1, 3, tzinfo=datetime.UTC)), - ObjectStorageItem( - key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.UTC) - ), + ObjectStorageItem(key="not_used/and_new", last_modified=datetime.datetime(2020, 1, 4, tzinfo=datetime.UTC)), ] ) manifests = [ @@ -1468,7 +1459,7 @@ async def test_get_versions_step() -> None: @pytest.mark.parametrize( - "versions,called_node_indicies", + ("versions", "called_node_indicies"), [ ([(23, 8), (23, 12)], [0, 1]), ([(23, 3), (23, 8)], [0]), @@ -1505,7 +1496,7 @@ def create_object_storage_disk(name: str, object_storage: ObjectStorage | None) @pytest.mark.parametrize( - "original_query,rewritten_query", + ("original_query", "rewritten_query"), [ [ b"CREATE VIEW FOO AS SELECT 1", diff --git a/tests/unit/coordinator/plugins/test_base.py b/tests/unit/coordinator/plugins/test_base.py index 9244b199..fcdeae0b 100644 --- a/tests/unit/coordinator/plugins/test_base.py +++ b/tests/unit/coordinator/plugins/test_base.py @@ -132,7 +132,7 @@ def manifest_with_hashes(hashes: dict[str, bytes], index: int) -> ipc.BackupMani @pytest.mark.parametrize( - "node_features,expected_request", + ("node_features", "expected_request"), [ ( [], @@ -195,7 +195,7 @@ async def test_upload_step_uses_new_request_if_supported( @pytest.mark.parametrize( - "retention,kept_backups", + ("retention", "kept_backups"), [ (ipc.Retention(minimum_backups=1, maximum_backups=6, keep_days=6), {"b1", "b2", "b3", "b4", "b5"}), (ipc.Retention(minimum_backups=1, maximum_backups=3, keep_days=6), {"b3", "b4", "b5"}), @@ -219,7 +219,7 @@ async def test_compute_kept_backups( ) context.set_result(ListBackupsStep, set(await async_json_storage.list_jsons())) half_a_day_after_last_backup = datetime.datetime(2020, 1, 7, 5, 0, tzinfo=datetime.UTC) - with mock.patch.object(utils, "now", lambda: half_a_day_after_last_backup): + with mock.patch.object(utils, "now", return_value=half_a_day_after_last_backup): result = await step.run_step(cluster=single_node_cluster, context=context) assert {b.filename for b in result} == kept_backups @@ -239,7 +239,7 @@ async def test_compute_kept_backups( @pytest.mark.parametrize( - "explicit_delete,expected_kept_backups", + ("explicit_delete", "expected_kept_backups"), [ ([], {"b1", "b2", "delta-231", "delta-232", "b3", "b4", "delta-451", "b5", "delta-561"}), (["b1"], {"b2", "delta-231", "delta-232", "b3", "b4", "delta-451", "b5", "delta-561"}), @@ -333,11 +333,10 @@ async def test_delete_backup_and_delta_manifests( @pytest.mark.parametrize( - "kept_backups,stored_hashes,expected_hashes", + ("kept_backups", "stored_hashes", "expected_hashes"), [ ([], {}, {}), ([], {"a": b"a"}, {}), - ([], {"a": b"a"}, {}), ( [manifest_with_hashes({"a": b"a"}, 0), manifest_with_hashes({"b": b"b"}, 1)], {"a": b"a", "b": b"b", "c": b"c"}, @@ -388,7 +387,7 @@ async def test_upload_manifest_step_generates_correct_backup_name( @pytest.mark.parametrize( - "node_features,expected_request", + ("node_features", "expected_request"), [ ([], None), ( diff --git a/tests/unit/coordinator/test_backup.py b/tests/unit/coordinator/test_backup.py index 6406238a..a36aa284 100644 --- a/tests/unit/coordinator/test_backup.py +++ b/tests/unit/coordinator/test_backup.py @@ -106,7 +106,7 @@ def _ssresults(*kwarg_list) -> list[ipc.SnapshotResult]: @pytest.mark.parametrize( - "hexdigests,snapshot_results,uploads", + ("hexdigests", "snapshot_results", "uploads"), [ ([], _ssresults({}, {}, {}, {}), []), ( diff --git a/tests/unit/coordinator/test_restore.py b/tests/unit/coordinator/test_restore.py index f27c1eaf..25bdf9e9 100644 --- a/tests/unit/coordinator/test_restore.py +++ b/tests/unit/coordinator/test_restore.py @@ -171,7 +171,7 @@ def match_clear(request: httpx.Request) -> httpx.Response | None: @pytest.mark.parametrize( - "node_azlist,backup_azlist,expected_index,exception", + ("node_azlist", "backup_azlist", "expected_index", "exception"), [ # successful cases ([], [], [], does_not_raise()), @@ -196,7 +196,7 @@ def test_node_to_backup_index( @pytest.mark.parametrize( - "partial_node_spec,expected_index,exception", + ("partial_node_spec", "expected_index", "exception"), [ # 4 (supported) ways of expressing same thing ({"backup_index": 1, "node_index": 2}, [None, None, 1], does_not_raise()), diff --git a/tests/unit/node/conftest.py b/tests/unit/node/conftest.py index 549228cd..7022110d 100644 --- a/tests/unit/node/conftest.py +++ b/tests/unit/node/conftest.py @@ -105,9 +105,9 @@ def build_snapshot_and_snapshotter( if snapshot_cls is SQLiteSnapshot: snapshot = SQLiteSnapshot(dst, db) snapshotter = SQLiteSnapshotter(src=src, dst=dst, snapshot=snapshot, groups=groups, parallel=2) + return snapshot, snapshotter else: - assert False - return snapshot, snapshotter + pytest.fail("Expected SQLite snapshotter class") def create_files_at_path(dir_: Path, files: list[tuple[str, bytes]]) -> None: diff --git a/tests/unit/node/test_node_snapshot.py b/tests/unit/node/test_node_snapshot.py index e3a02a1a..e3d39899 100644 --- a/tests/unit/node/test_node_snapshot.py +++ b/tests/unit/node/test_node_snapshot.py @@ -158,7 +158,7 @@ def _fun(self): @pytest.mark.timeout(2) @pytest.mark.parametrize( - "truncate_to,hashes_in_second_snapshot", + ("truncate_to", "hashes_in_second_snapshot"), [ (magic.DEFAULT_EMBEDDED_FILE_SIZE - 1, 0), (magic.DEFAULT_EMBEDDED_FILE_SIZE + 1, 1), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ab8a827b..a2a2e878 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize( - "version_command_output,expected_version", + ("version_command_output", "expected_version"), [ (b"ClickHouse server version 22.0.0.1 (official build).", (22, 0, 0, 1)), (b"ClickHouse server version 22.1.2.3.", (22, 1, 2, 3)), From 17e9778b87569b2f6a0987f4ecf638213e4d92e3 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 22:36:40 +0200 Subject: [PATCH 4/7] Remove pylint and flake8 --- .flake8 | 17 ----------------- .pre-commit-config.yaml | 21 --------------------- .pylintrc | 28 ---------------------------- Makefile | 13 +++---------- 4 files changed, 3 insertions(+), 76 deletions(-) delete mode 100644 .flake8 delete mode 100644 .pylintrc diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 26a71e4d..00000000 --- a/.flake8 +++ /dev/null @@ -1,17 +0,0 @@ -[flake8] -max-line-length = 125 -# E123: Closing brackets indent -# E126: Hanging indents -# E129: Visual indent -# E203: Whitespace before ':' (in accordance with ruff) -# E231: Missing whitespace after ',' (in accordance with ruff) -# E501: Max line length -# E722: bare-except -extend-ignore = - E123, - E126, - E129, - E203, - E231, - E501, - E722 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e637d47..4e5c5ab9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,27 +25,6 @@ repos: files: \.py$ exclude: ^vendor/ - - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 - hooks: - - id: flake8 - files: \.py$ - args: - - --config=.flake8 - - # https://pre-commit.com/#repository-local-hooks - - repo: local - hooks: - - id: pylint - name: pylint - entry: pylint - language: system - types: [python] - files: \.py$ - exclude: ^vendor/ - args: - - --rcfile=.pylintrc - - repo: local hooks: - id: mypy diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 2f899b1d..00000000 --- a/.pylintrc +++ /dev/null @@ -1,28 +0,0 @@ -[MESSAGES CONTROL] -disable= - duplicate-code, # the checker in 2.9.6 fails with parallelism - invalid-name, # fastapi conventions break this - missing-docstring, - no-member, # broken with pydantic + inheritance - too-few-public-methods, # some pydantic models have 0 and it is fine - too-many-arguments, - too-many-instance-attributes, # give me a break, <= 7 - too-many-lines, - too-many-locals, # locals make the code more readable - unused-argument, # annoying when fullfilling some API - use-implicit-booleaness-not-comparison, # leads to unclear code - unspecified-encoding, # irrelevant since the library is linux-specific - wrong-import-order, # we use isort anyway - -[FORMAT] -max-line-length=125 - -[REPORTS] -output-format=text -reports=no -score=no - -[TYPECHECK] -ignored-classes=responses -ignored-modules=cassandra -extension-pkg-whitelist=pydantic,cassandra.cluster,cassandra.metadata,cassandra.query diff --git a/Makefile b/Makefile index e56a44f8..715b0f67 100644 --- a/Makefile +++ b/Makefile @@ -40,13 +40,6 @@ test-dep-ubuntu: tar vxf apache-zookeeper-$(ZOOKEEPER_VERSION)-bin.tar.gz --wildcards apache-zookeeper-$(ZOOKEEPER_VERSION)-bin/lib/*.jar sudo cp -r apache-zookeeper-$(ZOOKEEPER_VERSION)-bin/lib /usr/share/zookeeper -.PHONY: pylint -pylint: $(GENERATED) - pre-commit run pylint --all-files - -.PHONY: flake8 -flake8: $(GENERATED) - pre-commit run flake8 --all-files .PHONY: copyright copyright: @@ -69,15 +62,15 @@ unittest: $(GENERATED) test: lint copyright unittest .PHONY: isort -isort: +isort: $(GENERATED) pre-commit run isort --all-files .PHONY: ruff -ruff: +ruff: $(GENERATED) pre-commit run ruff-format --all-files .PHONY: mypy -mypy: +mypy: $(GENERATED) pre-commit run mypy --all-files .PHONY: reformat From c5b94c62c6e88c39e8ca06f23ddae2549c19cb4b Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 22:38:31 +0200 Subject: [PATCH 5/7] Remove pylint ignore comments --- astacus/common/ipc.py | 1 - astacus/common/op.py | 4 ++-- astacus/common/rohmustorage.py | 2 +- astacus/common/storage.py | 1 - astacus/coordinator/plugins/__init__.py | 2 -- astacus/node/api.py | 9 +++------ astacus/server.py | 6 +++--- tests/plugins/asyncio_loop.py | 8 ++++---- tests/system/conftest.py | 1 - tests/unit/common/cassandra/test_schema.py | 2 -- tests/unit/common/test_op.py | 2 +- .../coordinator/plugins/cassandra/test_backup_steps.py | 2 -- .../coordinator/plugins/cassandra/test_restore_steps.py | 2 +- tests/unit/coordinator/plugins/clickhouse/test_steps.py | 2 +- tests/unit/coordinator/test_restore.py | 1 - tests/unit/test_client.py | 2 +- 16 files changed, 17 insertions(+), 30 deletions(-) diff --git a/astacus/common/ipc.py b/astacus/common/ipc.py index 43dd7eb1..a82315e9 100644 --- a/astacus/common/ipc.py +++ b/astacus/common/ipc.py @@ -17,7 +17,6 @@ import socket # pydantic validators are class methods in disguise -# pylint: disable=no-self-argument # These are the database plugins we support; list is intentionally diff --git a/astacus/common/op.py b/astacus/common/op.py index 643a1d9e..a45d94bd 100644 --- a/astacus/common/op.py +++ b/astacus/common/op.py @@ -121,7 +121,7 @@ async def _async_wrapper(): except asyncio.CancelledError: with contextlib.suppress(ExpiredOperationException): op.set_status_fail() - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: logger.warning("Unexpected exception during async %s %s %r", op, fun, ex) with contextlib.suppress(ExpiredOperationException): op.set_status_fail() @@ -134,7 +134,7 @@ def _sync_wrapper(): op.set_status(Op.Status.done, from_status=Op.Status.running) except ExpiredOperationException: pass - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: logger.warning("Unexpected exception during sync %s %s %r", op, fun, ex) with contextlib.suppress(ExpiredOperationException): op.set_status_fail() diff --git a/astacus/common/rohmustorage.py b/astacus/common/rohmustorage.py index 557b18d0..29606823 100644 --- a/astacus/common/rohmustorage.py +++ b/astacus/common/rohmustorage.py @@ -108,7 +108,7 @@ def _f(*a, **kw): return fun(*a, **kw) except errors.FileNotFoundFromStorageError as ex: raise exceptions.NotFoundException from ex - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: raise exceptions.RohmuException from ex return _f diff --git a/astacus/common/storage.py b/astacus/common/storage.py index 4cf16b6b..5e023e13 100644 --- a/astacus/common/storage.py +++ b/astacus/common/storage.py @@ -95,7 +95,6 @@ def download_json(self, name, struct_type: type[ST]) -> ST: class Storage(HexDigestStorage, JsonStorage, ABC): - # pylint: disable=abstract-method # This is abstract class which has whatever APIs necessary. Due to that, # it is expected not to implement the abstract methods. @abstractmethod diff --git a/astacus/coordinator/plugins/__init__.py b/astacus/coordinator/plugins/__init__.py index 383eb7c3..ae805616 100644 --- a/astacus/coordinator/plugins/__init__.py +++ b/astacus/coordinator/plugins/__init__.py @@ -3,8 +3,6 @@ def get_plugin(plugin: Plugin) -> type[CoordinatorPlugin]: - # pylint: disable=import-outside-toplevel - if plugin == Plugin.cassandra: from .cassandra.plugin import CassandraPlugin diff --git a/astacus/node/api.py b/astacus/node/api.py index cf55bd32..9d6e8929 100644 --- a/astacus/node/api.py +++ b/astacus/node/api.py @@ -310,8 +310,7 @@ def cassandra_start_cassandra( replace_address_first_boot=replace_address_first_boot, skip_bootstrap_streaming=skip_bootstrap_streaming, ) - # pylint: disable=import-outside-toplevel - # pylint: disable=raise-missing-from + try: from .cassandra import CassandraStartOp except ImportError: @@ -336,8 +335,7 @@ def cassandra_restore_sstables( match_tables_by=match_tables_by, expect_empty_target=expect_empty_target, ) - # pylint: disable=import-outside-toplevel - # pylint: disable=raise-missing-from + try: from .cassandra import CassandraRestoreSSTablesOp except ImportError: @@ -349,8 +347,7 @@ def cassandra_restore_sstables( @router.post("/cassandra/{subop}") def cassandra(subop: ipc.CassandraSubOp, result_url: Annotated[str, Body(embed=True)] = "", n: Node = Depends()): req = ipc.NodeRequest(result_url=result_url) - # pylint: disable=import-outside-toplevel - # pylint: disable=raise-missing-from + try: from .cassandra import CassandraGetSchemaHashOp, SimpleCassandraSubOp except ImportError: diff --git a/astacus/server.py b/astacus/server.py index 9806d6ed..b6ddb916 100644 --- a/astacus/server.py +++ b/astacus/server.py @@ -53,9 +53,9 @@ async def _shutdown_event(): gconfig = config.set_global_config_from_path(api, config_path) sentry_dsn = os.environ.get("SENTRY_DSN", gconfig.sentry_dsn) if sentry_dsn: - sentry_sdk.init(dsn=sentry_dsn) # pylint: disable=abstract-class-instantiated + sentry_sdk.init(dsn=sentry_dsn) api.add_middleware(SentryAsgiMiddleware) - global app # pylint: disable=global-statement + global app app = api return api @@ -68,7 +68,7 @@ def _systemd_notify_ready(): if not os.environ.get("NOTIFY_SOCKET"): return try: - from systemd import daemon # pylint: disable=no-name-in-module,disable=import-outside-toplevel + from systemd import daemon daemon.notify("READY=1") except ImportError: diff --git a/tests/plugins/asyncio_loop.py b/tests/plugins/asyncio_loop.py index 34411f7d..f1c78a95 100644 --- a/tests/plugins/asyncio_loop.py +++ b/tests/plugins/asyncio_loop.py @@ -79,7 +79,7 @@ def finalizer() -> Any: def pytest_pycollect_makeitem(collector: PyCollector, name: str, obj: Any) -> list[Function] | None: """Auto-add a "loop" fixture to all async test functions.""" if collector.funcnamefilter(name) and asyncio.iscoroutinefunction(obj): - functions = list(collector._genfunctions(name, obj)) # pylint: disable=protected-access + functions = list(collector._genfunctions(name, obj)) for function in functions: if "loop" not in function.fixturenames: function.fixturenames.append("loop") @@ -91,7 +91,7 @@ def pytest_pyfunc_call(pyfuncitem: Function) -> bool | None: """Run coroutines in an event loop instead of a normal function call.""" if asyncio.iscoroutinefunction(pyfuncitem.function): with _runtime_warning_context(): - fixture_info = pyfuncitem._fixtureinfo # pylint: disable=protected-access + fixture_info = pyfuncitem._fixtureinfo test_args = {arg: pyfuncitem.funcargs[arg] for arg in fixture_info.argnames} loop = cast(asyncio.AbstractEventLoop, pyfuncitem.funcargs["loop"]) loop.run_until_complete(pyfuncitem.obj(**test_args)) @@ -161,7 +161,7 @@ def pytest_collection_modifyitems(session: pytest.Session, config: Config, items def get_scope_identifiers(item: pytest.Function) -> Iterator[Sequence[str]]: """Enumerate all scopes of all the async fixtures required for a test function.""" - fixture_info = item._fixtureinfo # pylint: disable=protected-access + fixture_info = item._fixtureinfo for fixture_name in fixture_info.initialnames: if fixture_name == "request": continue @@ -185,7 +185,7 @@ def get_scope_identifiers(item: pytest.Function) -> Iterator[Sequence[str]]: def get_loop(request: SubRequest) -> Iterator[asyncio.AbstractEventLoop]: """Create a new async loop or reuse one from the outermost async scope.""" - tested_function_request = request._pyfuncitem._request # pylint: disable=protected-access + tested_function_request = request._pyfuncitem._request async_scopes = tested_function_request.node.async_scope.connected_scopes for scope in FIXTURE_SCOPES: if scope == request.scope: diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 74c2128e..c6ae6cc1 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -49,7 +49,6 @@ class TestNode(AstacusModel): @asynccontextmanager async def background_process(program: str | Path, *args: str | Path, **kwargs) -> AsyncIterator[asyncio.subprocess.Process]: - # pylint: disable=bare-except proc = await asyncio.create_subprocess_exec(program, *args, **kwargs) try: yield proc diff --git a/tests/unit/common/cassandra/test_schema.py b/tests/unit/common/cassandra/test_schema.py index 62544d45..7d157eca 100644 --- a/tests/unit/common/cassandra/test_schema.py +++ b/tests/unit/common/cassandra/test_schema.py @@ -9,8 +9,6 @@ import pytest -# pylint: disable=protected-access - def test_schema(mocker: MockerFixture) -> None: cut = schema.CassandraUserType(name="cut", cql_create_self="CREATE-USER-TYPE", field_types=["type1", "type2"]) diff --git a/tests/unit/common/test_op.py b/tests/unit/common/test_op.py index e4e3ffc8..ba251273 100644 --- a/tests/unit/common/test_op.py +++ b/tests/unit/common/test_op.py @@ -51,7 +51,7 @@ async def _async(): else: mixin.start_op(op=op_obj, op_name="dummy", fun=_sync) await mixin.background_tasks() - except Exception as ex: # pylint: disable=broad-except + except Exception as ex: assert expect_ex assert isinstance(ex, expect_ex) assert op_obj.info.op_status == expect_status diff --git a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py index 954abdb2..def8e24d 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py @@ -2,8 +2,6 @@ See LICENSE for details. """ -# pylint: disable=protected-access - from astacus.common.cassandra.schema import CassandraSchema from astacus.coordinator.plugins.cassandra import backup_steps from astacus.coordinator.plugins.cassandra.model import CassandraConfigurationNode diff --git a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py index 267dbcb4..dfed0ad2 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py @@ -211,7 +211,7 @@ def test_rewrite_datacenters() -> None: .with_cql_create_self(pre_rewrite_cql) .with_network_topology_strategy_dcs({"new_dc": "3"}), ] - restore_steps._rewrite_datacenters(keyspaces) # pylint: disable=protected-access + restore_steps._rewrite_datacenters(keyspaces) unchanged_keyspace, rewritten_keyspace = keyspaces[0], keyspaces[1] assert unchanged_keyspace.cql_create_self == pre_rewrite_cql assert "'new_dc': '3'" in rewritten_keyspace.cql_create_self diff --git a/tests/unit/coordinator/plugins/clickhouse/test_steps.py b/tests/unit/coordinator/plugins/clickhouse/test_steps.py index 1defe3a2..3cea6f05 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_steps.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_steps.py @@ -85,7 +85,7 @@ from tests.unit.storage import MemoryJsonStorage from typing import Any from unittest import mock -from unittest.mock import _Call as MockCall, patch # pylint: disable=protected-access +from unittest.mock import _Call as MockCall, patch import asyncio import base64 diff --git a/tests/unit/coordinator/test_restore.py b/tests/unit/coordinator/test_restore.py index 25bdf9e9..10517ce5 100644 --- a/tests/unit/coordinator/test_restore.py +++ b/tests/unit/coordinator/test_restore.py @@ -71,7 +71,6 @@ class RestoreTest: ], ) def test_restore(rt: RestoreTest, app: FastAPI, client: TestClient, tmp_path: Path) -> None: - # pylint: disable=too-many-statements # Create fake backup (not pretty but sufficient?) storage_factory = StorageFactory( storage_config=app.state.coordinator_config.object_storage, diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 40c71e15..1ea1ab2e 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -46,7 +46,7 @@ def sleep(duration: float) -> None: with mock.patch.object(time, "sleep", new=sleep): # mypy wants a return for this function but pylint doesn't - # pylint: disable=useless-return + def http_request(*args, timeout: float = 10, **kwargs) -> Mapping[str, Any] | None: time_since_start = time.monotonic() - start_time assert time_since_start + timeout <= wait_completion_secs, "request could end after completion" From 146ee438c09482315bc9dbffcebf0525341dc7d9 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Sat, 19 Oct 2024 22:46:38 +0200 Subject: [PATCH 6/7] Remove IDE compatiblility directive --- astacus/coordinator/plugins/clickhouse/dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astacus/coordinator/plugins/clickhouse/dependencies.py b/astacus/coordinator/plugins/clickhouse/dependencies.py index 994208bf..4e52433a 100644 --- a/astacus/coordinator/plugins/clickhouse/dependencies.py +++ b/astacus/coordinator/plugins/clickhouse/dependencies.py @@ -6,7 +6,6 @@ from collections.abc import Callable, Hashable, Sequence from typing import TypeVar -# noinspection PyCompatibility import graphlib import re import uuid From 3fc21c1341e21a5b012b9462fa4e99b6bc3e91c3 Mon Sep 17 00:00:00 2001 From: Aris Tritas Date: Thu, 24 Oct 2024 12:03:21 +0200 Subject: [PATCH 7/7] Remove redundant isort pre-commit check It is superseded by a ruff rule --- .pre-commit-config.yaml | 7 ------- Makefile | 8 +------- pyproject.toml | 8 -------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e5c5ab9..b1a091eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,13 +18,6 @@ repos: # Run the formatter. - id: ruff-format - - repo: https://github.com/timothycrosley/isort - rev: 5.13.2 - hooks: - - id: isort - files: \.py$ - exclude: ^vendor/ - - repo: local hooks: - id: mypy diff --git a/Makefile b/Makefile index 715b0f67..45a16a43 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ default: $(GENERATED) venv: default python3 -m venv venv && source venv/bin/activate && pip install -U pip && pip install . ".[dev]" +.PHONY: clean clean: rm -rf rpm/ $(GENERATED) venv/ .mypy_cache/ astacus.egg-info/ @@ -61,10 +62,6 @@ unittest: $(GENERATED) .PHONY: test test: lint copyright unittest -.PHONY: isort -isort: $(GENERATED) - pre-commit run isort --all-files - .PHONY: ruff ruff: $(GENERATED) pre-commit run ruff-format --all-files @@ -73,9 +70,6 @@ ruff: $(GENERATED) mypy: $(GENERATED) pre-commit run mypy --all-files -.PHONY: reformat -reformat: isort ruff - .PHONY: pre-commit pre-commit: $(GENERATED) pre-commit run --all-files diff --git a/pyproject.toml b/pyproject.toml index 8350f533..82ae5ed3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,14 +127,6 @@ astacus = "astacus.main:main" [tool.hatch.version] source = "vcs" -[tool.isort] -no_sections = true -force_alphabetical_sort = true -combine_as_imports = true -profile = "black" -skip_gitignore = true -line_length = 125 - [tool.hatch.build.hooks.vcs] version-file = "astacus/version.py"