diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 327322b..78564cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Analysing the code with pylint run: | - python -m pylint -E partitionmanager + python -m pylint --errors-only partitionmanager - name: Lint Python code with Ruff run: | diff --git a/partitionmanager/cli.py b/partitionmanager/cli.py index ed76786..b40b131 100644 --- a/partitionmanager/cli.py +++ b/partitionmanager/cli.py @@ -182,8 +182,8 @@ def _extract_single_column(row): def list_tables(conf): """List all tables for the current database.""" rows = conf.dbcmd.run("SHOW TABLES;") - table_names = map(lambda row: _extract_single_column(row), rows) - table_objects = map(lambda name: partitionmanager.types.Table(name), table_names) + table_names = (_extract_single_column(row) for row in rows) + table_objects = (partitionmanager.types.Table(name) for name in table_names) return list(table_objects) @@ -287,7 +287,7 @@ def do_partition(conf): log.info("Database is read-only, only emitting statistics") if conf.prometheus_stats_path: do_stats(conf) - return dict() + return {} if conf.noop: log.info("Running in noop mode, no changes will be made") @@ -304,7 +304,7 @@ def do_partition(conf): type_name="counter", ) - all_results = dict() + all_results = {} for table in conf.tables: time_start = None try: @@ -384,7 +384,7 @@ def do_stats(conf, metrics=partitionmanager.stats.PrometheusMetrics()): log = logging.getLogger("do_stats") - all_results = dict() + all_results = {} for table in list_tables(conf): table_problems = pm_tap.get_table_compatibility_problems(conf.dbcmd, table) if table_problems: @@ -476,7 +476,7 @@ def drop_cmd(args): def do_find_drops_for_tables(conf): - all_results = dict() + all_results = {} for table in conf.tables: log = logging.getLogger(f"do_find_drops_for_tables:{table.name}") diff --git a/partitionmanager/cli_test.py b/partitionmanager/cli_test.py index 3d658c2..3fd7edc 100644 --- a/partitionmanager/cli_test.py +++ b/partitionmanager/cli_test.py @@ -127,7 +127,7 @@ def test_partition_cmd_several_tables(self): output = partition_cmd(args) self.assertEqual(len(output), 2) - self.assertSetEqual(set(output), set(["testtable", "another_table"])) + self.assertSetEqual(set(output), {"testtable", "another_table"}) def test_partition_unpartitioned_table(self): o = run_partition_cmd_yaml( @@ -186,7 +186,7 @@ def test_partition_cmd_two_tables(self): mariadb: {str(fake_exec)} """ ) - self.assertSetEqual(set(o), set(["test", "test_with_retention"])) + self.assertSetEqual(set(o), {"test", "test_with_retention"}) def test_partition_period_daily(self): o = run_partition_cmd_yaml( @@ -201,7 +201,7 @@ def test_partition_period_daily(self): """ ) self.assertSequenceEqual( - set(o), set(["partitioned_last_week", "partitioned_yesterday"]) + set(o), {"partitioned_last_week", "partitioned_yesterday"} ) def test_partition_period_seven_days(self): @@ -221,16 +221,14 @@ def test_partition_period_seven_days(self): self.assertEqual( set(logctx.output), - set( - [ - "INFO:partition:Evaluating Table partitioned_last_week " - "(duration=7 days, 0:00:00)", - "DEBUG:partition:Table partitioned_last_week has no pending SQL updates.", # noqa: E501 - "INFO:partition:Evaluating Table partitioned_yesterday " - "(duration=7 days, 0:00:00)", - "DEBUG:partition:Table partitioned_yesterday has no pending SQL updates.", # noqa: E501 - ] - ), + { + "INFO:partition:Evaluating Table partitioned_last_week " + "(duration=7 days, 0:00:00)", + "DEBUG:partition:Table partitioned_last_week has no pending SQL updates.", # noqa: E501 + "INFO:partition:Evaluating Table partitioned_yesterday " + "(duration=7 days, 0:00:00)", + "DEBUG:partition:Table partitioned_yesterday has no pending SQL updates.", # noqa: E501 + }, ) self.assertSequenceEqual(list(o), []) @@ -249,7 +247,7 @@ def test_partition_period_different_per_table(self): """ ) self.assertSequenceEqual( - set(o), set(["partitioned_yesterday", "partitioned_last_week"]) + set(o), {"partitioned_yesterday", "partitioned_last_week"} ) def test_partition_with_db_url(self): @@ -283,7 +281,7 @@ def assert_stats_results(self, results): def assert_stats_prometheus_outfile(self, prom_file): lines = prom_file.split("\n") - metrics = dict() + metrics = {} for line in lines: if not line.startswith("#") and len(line) > 0: key, value = line.split(" ") @@ -350,9 +348,7 @@ def test_cli_tables_override_yaml(self): """, datetime.now(tz=timezone.utc), ) - self.assertEqual( - {str(x.name) for x in conf.tables}, set(["table_one", "table_two"]) - ) + self.assertEqual({str(x.name) for x in conf.tables}, {"table_one", "table_two"}) def test_cli_mariadb_override_yaml(self): args = PARSER.parse_args(["--mariadb", "/usr/bin/true", "stats"]) @@ -651,12 +647,10 @@ def test_drop_invalid_config(self): ) self.assertEqual( set(logctx.output), - set( - [ - "WARNING:do_find_drops_for_tables:unused:" - "Cannot process Table unused: no retention specified" - ] - ), + { + "WARNING:do_find_drops_for_tables:unused:" + "Cannot process Table unused: no retention specified" + }, ) def test_drop_no_sql(self): @@ -675,10 +669,8 @@ def test_drop_no_sql(self): ) self.assertEqual( set(logctx.output), - set( - [ - "WARNING:do_find_drops_for_tables:unused:" - "Cannot process Table unused: no date query specified" - ] - ), + { + "WARNING:do_find_drops_for_tables:unused:" + "Cannot process Table unused: no date query specified" + }, ) diff --git a/partitionmanager/database_helpers_test.py b/partitionmanager/database_helpers_test.py index 280ae17..b7a3a17 100644 --- a/partitionmanager/database_helpers_test.py +++ b/partitionmanager/database_helpers_test.py @@ -15,7 +15,7 @@ class MockDatabase(DatabaseCommand): def __init__(self): - self._responses = list() + self._responses = [] self.num_queries = 0 def add_response(self, expected, response): diff --git a/partitionmanager/dropper.py b/partitionmanager/dropper.py index df24c15..2dc68bf 100644 --- a/partitionmanager/dropper.py +++ b/partitionmanager/dropper.py @@ -16,7 +16,7 @@ def _drop_statement(table, partition_list): if not partition_list: raise ValueError("Partition list may not be empty") - partitions = ",".join(map(lambda x: f"`{x.name}`", partition_list)) + partitions = ",".join(f"`{x.name}`" for x in partition_list) alter_cmd = f"ALTER TABLE `{table.name}` DROP PARTITION IF EXISTS {partitions};" diff --git a/partitionmanager/dropper_test.py b/partitionmanager/dropper_test.py index d1eb8b7..f6b4ecc 100644 --- a/partitionmanager/dropper_test.py +++ b/partitionmanager/dropper_test.py @@ -20,7 +20,7 @@ def _timestamp_rsp(year, mo, day): class MockDatabase(DatabaseCommand): def __init__(self): - self._responses = list() + self._responses = [] self.num_queries = 0 def add_response(self, expected, response): diff --git a/partitionmanager/migrate.py b/partitionmanager/migrate.py index 00a85ca..9fc3d0c 100644 --- a/partitionmanager/migrate.py +++ b/partitionmanager/migrate.py @@ -48,7 +48,7 @@ def write_state_info(conf, out_fp): log = logging.getLogger("write_state_info") log.info("Writing current state information") - state_info = {"time": conf.curtime, "tables": dict()} + state_info = {"time": conf.curtime, "tables": {}} for table in conf.tables: map_data = _get_map_data_from_config(conf, table) @@ -90,7 +90,7 @@ def _plan_partitions_for_time_offsets( rate_of_change: an ordered list of positions per RATE_UNIT. """ - changes = list() + changes = [] for (i, offset), is_final in partitionmanager.tools.iter_show_end( enumerate(time_offsets) ): @@ -243,7 +243,7 @@ def calculate_sql_alters_from_state_info(conf, in_fp): f"{prior_data['time']} = {time_delta}" ) - commands = dict() + commands = {} for table_name, prior_pos in prior_data["tables"].items(): table = None @@ -270,7 +270,7 @@ def calculate_sql_alters_from_state_info(conf, in_fp): delta_positions = list( map(operator.sub, ordered_current_pos, ordered_prior_pos) ) - rate_of_change = list(map(lambda pos: pos / time_delta, delta_positions)) + rate_of_change = [pos / time_delta for pos in delta_positions] max_val_part = map_data["partitions"][-1] if not isinstance(max_val_part, partitionmanager.types.MaxValuePartition): diff --git a/partitionmanager/migrate_test.py b/partitionmanager/migrate_test.py index 8aee422..23dee04 100644 --- a/partitionmanager/migrate_test.py +++ b/partitionmanager/migrate_test.py @@ -26,7 +26,7 @@ class MockDatabase(DatabaseCommand): def __init__(self): - self._response = list() + self._response = [] self._select_response = [[{"id": 150}]] self.num_queries = 0 diff --git a/partitionmanager/sql.py b/partitionmanager/sql.py index aaae34f..0516834 100644 --- a/partitionmanager/sql.py +++ b/partitionmanager/sql.py @@ -53,7 +53,7 @@ def __init__(self): self.rows = None self.current_row = None self.current_field = None - self.current_elements = list() + self.current_elements = [] self.statement = None def parse(self, data): @@ -61,7 +61,7 @@ def parse(self, data): if self.rows is not None: raise ValueError("XmlResult objects can only be used once") - self.rows = list() + self.rows = [] self.xmlparser.Parse(data) if self.current_elements: @@ -186,4 +186,4 @@ def run(self, sql_cmd): logging.debug(f"IntegratedDatabaseCommand executing {sql_cmd}") with self.connection.cursor() as cursor: cursor.execute(sql_cmd) - return [row for row in cursor] + return list(cursor) diff --git a/partitionmanager/stats.py b/partitionmanager/stats.py index 13a0196..f186cc0 100644 --- a/partitionmanager/stats.py +++ b/partitionmanager/stats.py @@ -22,14 +22,14 @@ class PrometheusMetrics: """A set of metrics that can be rendered for Prometheus.""" def __init__(self): - self.metrics = dict() - self.help = dict() - self.types = dict() + self.metrics = {} + self.help = {} + self.types = {} def add(self, name, table, data): """Record metric data representing the name and table.""" if name not in self.metrics: - self.metrics[name] = list() + self.metrics[name] = [] self.metrics[name].append(PrometheusMetric(name, table, data)) def describe(self, name, help_text=None, type_name=None): @@ -50,7 +50,7 @@ def render(self, fp): if n in self.types: print(f"# TYPE {name} {self.types[n]}", file=fp) for m in metrics: - labels = list() + labels = [] if m.table: labels = [f'table="{m.table}"'] print(f"{name}{{{','.join(labels)}}} {m.data}", file=fp) diff --git a/partitionmanager/stats_test.py b/partitionmanager/stats_test.py index 9bfad2c..6e9f4bc 100644 --- a/partitionmanager/stats_test.py +++ b/partitionmanager/stats_test.py @@ -11,7 +11,7 @@ class TestStatistics(unittest.TestCase): def test_statistics_no_partitions(self): - s = get_statistics(list(), ts, Table("no_parts")) + s = get_statistics([], ts, Table("no_parts")) self.assertEqual(s, {"partitions": 0}) def test_statistics_single_unnamed_partition(self): @@ -44,7 +44,7 @@ def test_statistics_two_partitions(self): ) def test_statistics_weekly_partitions_year(self): - parts = list() + parts = [] base = datetime(2020, 5, 20, tzinfo=timezone.utc) for w in range(52): partName = f"p_{base + timedelta(weeks=w):%Y%m%d}" diff --git a/partitionmanager/table_append_partition.py b/partitionmanager/table_append_partition.py index 7d2c7e8..9a87e52 100644 --- a/partitionmanager/table_append_partition.py +++ b/partitionmanager/table_append_partition.py @@ -37,7 +37,7 @@ def _get_table_information_schema_problems(rows, table_name): options = rows[0] if "partitioned" not in options["CREATE_OPTIONS"]: return [f"Table {table_name} is not partitioned"] - return list() + return [] def get_current_positions(database, table, columns): @@ -50,7 +50,7 @@ def get_current_positions(database, table, columns): ): raise ValueError("columns must be a list and table must be a Table") - positions = dict() + positions = {} for column in columns: if not isinstance(column, str): raise ValueError("columns must be a list of strings") @@ -100,7 +100,7 @@ def _parse_partition_map(rows): ) range_cols = None - partitions = list() + partitions = [] if len(rows) != 1: raise partitionmanager.types.TableInformationException("Expected one result") @@ -203,8 +203,8 @@ def _split_partitions_around_position(partition_list, current_position): if not isinstance(current_position, partitionmanager.types.Position): raise ValueError - less_than_partitions = list() - greater_or_equal_partitions = list() + less_than_partitions = [] + greater_or_equal_partitions = [] for p in partition_list: if p < current_position: @@ -239,20 +239,20 @@ def _get_position_increase_per_day(p1, p2): if None in (p1.timestamp(), p2.timestamp()): # An empty list skips this pair in get_weighted_position_increase - return list() + return [] if p1.timestamp() >= p2.timestamp(): log.warning( f"Skipping rate of change between p1 {p1} and p2 {p2} as they are " "out-of-order" ) - return list() + return [] delta_time = p2.timestamp() - p1.timestamp() delta_days = delta_time / timedelta(days=1) delta_positions = list( map(operator.sub, p2.position.as_list(), p1.position.as_list()) ) - return list(map(lambda pos: pos / delta_days, delta_positions)) + return [pos / delta_days for pos in delta_positions] def _generate_weights(count): @@ -295,7 +295,7 @@ def _get_weighted_position_increase_per_day_for_partitions(partitions): for p_r, weight in zip(pos_rates, weights): for idx, val in enumerate(p_r): weighted_sums[idx] += val * weight - return list(map(lambda x: x / sum(weights), weighted_sums)) + return [x / sum(weights) for x in weighted_sums] def _predict_forward_position(current_positions, rate_of_change, duration): @@ -313,7 +313,7 @@ def _predict_forward_position(current_positions, rate_of_change, duration): f"Can't predict forward with a negative rate of change: {neg_rate}" ) - increase = list(map(lambda x: x * (duration / timedelta(days=1)), rate_of_change)) + increase = [x * (duration / timedelta(days=1)) for x in rate_of_change] predicted_positions = [int(p + i) for p, i in zip(current_positions, increase)] for old, new in zip(current_positions, predicted_positions): assert new >= old, f"Always predict forward, {new} < {old}" @@ -431,7 +431,7 @@ def _get_rate_partitions_with_queried_timestamps( if not table.has_date_query: raise ValueError("Table has no defined date query") - instant_partitions = list() + instant_partitions = [] for partition in partition_list: exact_time = ( @@ -587,7 +587,7 @@ def _plan_partition_changes( while conflict_found: conflict_found = False - existing_timestamps = set(map(lambda p: p.timestamp(), partition_list)) + existing_timestamps = {p.timestamp() for p in partition_list} for partition in results: if partition.timestamp() in existing_timestamps: @@ -648,8 +648,8 @@ def generate_sql_reorganize_partition_commands(table, changes): """ log = logging.getLogger(f"generate_sql_reorganize_partition_commands:{table.name}") - modified_partitions = list() - new_partitions = list() + modified_partitions = [] + new_partitions = [] for p in changes: if isinstance(p, partitionmanager.types.ChangePlannedPartition): @@ -684,7 +684,7 @@ def generate_sql_reorganize_partition_commands(table, changes): log.debug(f"{modified_partition} does not have modifications, skip") continue - partition_strings = list() + partition_strings = [] for part in new_part_list: if part.name in partition_names_set: raise partitionmanager.types.DuplicatePartitionException( @@ -757,7 +757,7 @@ def get_pending_sql_reorganize_partition_commands( if not _should_run_changes(table, partition_changes): log.info(f"{table} does not need to be modified currently.") - return list() + return [] log.debug(f"{table} has changes waiting.") return generate_sql_reorganize_partition_commands(table, partition_changes) diff --git a/partitionmanager/table_append_partition_test.py b/partitionmanager/table_append_partition_test.py index ae771d7..b456da5 100644 --- a/partitionmanager/table_append_partition_test.py +++ b/partitionmanager/table_append_partition_test.py @@ -90,11 +90,11 @@ def test_not_partitioned(self): def test_normal(self): info = [{"CREATE_OPTIONS": "partitioned"}] - self.assertEqual(_get_table_information_schema_problems(info, "table"), list()) + self.assertEqual(_get_table_information_schema_problems(info, "table"), []) def test_normal_multiple_create_options(self): info = [{"CREATE_OPTIONS": "magical, partitioned"}] - self.assertEqual(_get_table_information_schema_problems(info, "table"), list()) + self.assertEqual(_get_table_information_schema_problems(info, "table"), []) class TestParsePartitionMap(unittest.TestCase): @@ -403,7 +403,7 @@ def test_generate_weights(self): def test_get_weighted_position_increase_per_day_for_partitions(self): with self.assertRaises(ValueError): - _get_weighted_position_increase_per_day_for_partitions(list()) + _get_weighted_position_increase_per_day_for_partitions([]) self.assertEqual( _get_weighted_position_increase_per_day_for_partitions( diff --git a/partitionmanager/types.py b/partitionmanager/types.py index dc4745c..7f31c74 100644 --- a/partitionmanager/types.py +++ b/partitionmanager/types.py @@ -254,7 +254,7 @@ class Position: """ def __init__(self): - self._position = list() + self._position = [] def set_position(self, position_in): """Set the list of identifiers for this position.""" diff --git a/partitionmanager/types_test.py b/partitionmanager/types_test.py index 198fdc4..1b1fa4c 100644 --- a/partitionmanager/types_test.py +++ b/partitionmanager/types_test.py @@ -165,7 +165,7 @@ def test_table(self): with self.assertRaises(argparse.ArgumentTypeError): timedelta_from_dict({"another thing": 1, "days": 30}) - r = timedelta_from_dict(dict()) + r = timedelta_from_dict({}) self.assertEqual(None, r) with self.assertRaises(TypeError): diff --git a/pyproject.toml b/pyproject.toml index 0224344..f0eccf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ select = [ "ASYNC", # flake8-async "B", # flake8-bugbear "BLE", # flake8-blind-except + "C4", # flake8-comprehensions "C90", # McCabe cyclomatic complexity "DJ", # flake8-django "DTZ", # flake8-datetimez @@ -83,7 +84,6 @@ select = [ "W", # pycodestyle "YTT", # flake8-2020 # "ANN", # flake8-annotations - # "C4", # flake8-comprehensions # "COM", # flake8-commas # "CPY", # flake8-copyright # "D", # pydocstyle @@ -100,7 +100,10 @@ select = [ # "T20", # flake8-print # "TRY", # tryceratops ] -ignore = ["S101"] # Allow assert statements +ignore = [ + "ISC001", # Implicit string concatenation can conflict with ruff format + "S101", # Allow assert statements +] [tool.ruff.lint.mccabe] max-complexity = 16 # default is 10