From 4e04e4bd4dd65ff8b0410175fc3682717606849d Mon Sep 17 00:00:00 2001 From: Maurits Kroese Date: Mon, 17 May 2021 22:25:12 +0200 Subject: [PATCH 1/9] Made the plugin a class so we can store state We need to able to store state so that we can report on the result of the splitting of the test suite in the corresponding hook. This commit makes the plugin a class without changing existing behaviour. --- src/pytest_split/plugin.py | 128 +++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 62 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index e87ff75..00af9c7 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -43,69 +43,73 @@ def pytest_addoption(parser): ) -def pytest_collection_modifyitems(session, config, items): - splits = config.option.splits - group = config.option.group - store_durations = config.option.store_durations - durations_report_path = config.option.durations_path - - if any((splits, group)): - if not all((splits, group)): - return - if not os.path.isfile(durations_report_path): - return - if store_durations: - # Don't split if we are storing durations - return - total_tests_count = len(items) - if splits and group: - with open(durations_report_path) as f: - stored_durations = OrderedDict(json.load(f)) - - start_idx, end_idx = _calculate_suite_start_and_end_idx( - splits, group, items, stored_durations - ) - items[:] = items[start_idx:end_idx] - - terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") - terminal_writer = create_terminal_writer(config) - message = terminal_writer.markup( - " Running group {}/{} ({}/{} tests)\n".format( - group, splits, len(items), total_tests_count +class SplitPlugin: + def pytest_collection_modifyitems(self, session, config, items): + splits = config.option.splits + group = config.option.group + store_durations = config.option.store_durations + durations_report_path = config.option.durations_path + + if any((splits, group)): + if not all((splits, group)): + return + if not os.path.isfile(durations_report_path): + return + if store_durations: + # Don't split if we are storing durations + return + total_tests_count = len(items) + if splits and group: + with open(durations_report_path) as f: + stored_durations = OrderedDict(json.load(f)) + + start_idx, end_idx = _calculate_suite_start_and_end_idx( + splits, group, items, stored_durations ) - ) - terminal_reporter.write(message) - - -def pytest_sessionfinish(session, exitstatus): - if session.config.option.store_durations: - report_path = session.config.option.durations_path - terminal_reporter = session.config.pluginmanager.get_plugin("terminalreporter") - durations = defaultdict(float) - for test_reports in terminal_reporter.stats.values(): - for test_report in test_reports: - if hasattr(test_report, "duration"): - stage = getattr(test_report, "when", "") - duration = test_report.duration - # These ifs be removed after this is solved: https://github.com/spulec/freezegun/issues/286 - if duration < 0: - continue - if ( - stage in ("teardown", "setup") - and duration > STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD - ): - # Ignore not legit teardown durations - continue - durations[test_report.nodeid] += test_report.duration - - with open(report_path, "w") as f: - f.write(json.dumps(list(durations.items()), indent=2)) - - terminal_writer = create_terminal_writer(session.config) - message = terminal_writer.markup( - " Stored test durations in {}\n".format(report_path) - ) - terminal_reporter.write(message) + items[:] = items[start_idx:end_idx] + + terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") + terminal_writer = create_terminal_writer(config) + message = terminal_writer.markup( + " Running group {}/{} ({}/{} tests)\n".format( + group, splits, len(items), total_tests_count + ) + ) + terminal_reporter.write(message) + + def pytest_sessionfinish(self, session, exitstatus): + if session.config.option.store_durations: + report_path = session.config.option.durations_path + terminal_reporter = session.config.pluginmanager.get_plugin("terminalreporter") + durations = defaultdict(float) + for test_reports in terminal_reporter.stats.values(): + for test_report in test_reports: + if hasattr(test_report, "duration"): + stage = getattr(test_report, "when", "") + duration = test_report.duration + # These ifs be removed after this is solved: https://github.com/spulec/freezegun/issues/286 + if duration < 0: + continue + if ( + stage in ("teardown", "setup") + and duration > STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD + ): + # Ignore not legit teardown durations + continue + durations[test_report.nodeid] += test_report.duration + + with open(report_path, "w") as f: + f.write(json.dumps(list(durations.items()), indent=2)) + + terminal_writer = create_terminal_writer(session.config) + message = terminal_writer.markup( + " Stored test durations in {}\n".format(report_path) + ) + terminal_reporter.write(message) + + +def pytest_configure(config): + config.pluginmanager.register(SplitPlugin()) def _calculate_suite_start_and_end_idx(splits, group, items, stored_durations): From f35e9093926430b0cd7130b46d150f6b30f75025 Mon Sep 17 00:00:00 2001 From: Maurits Kroese Date: Tue, 18 May 2021 21:50:05 +0200 Subject: [PATCH 2/9] Use pytest_report_collectionfinish The pytest_report_collectionfinish is the hook that gets called after item collection. That seems to be the right moment to report on the splitting result. This commit ensures pytest-split always reports how the test were split and uses the same message as before. --- src/pytest_split/plugin.py | 29 +++++++++++++++++++---------- tests/test_plugin.py | 9 +++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 00af9c7..7557acd 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -44,21 +44,38 @@ def pytest_addoption(parser): class SplitPlugin: + def __init__(self): + self._suite_num_tests: int + self._group_num_tests: int + + def pytest_report_collectionfinish(self, config): + return ( + f"Running group {config.option.group}/{config.option.splits}" + f" ({self._group_num_tests}/{self._suite_num_tests}) tests" + ) + def pytest_collection_modifyitems(self, session, config, items): splits = config.option.splits group = config.option.group store_durations = config.option.store_durations durations_report_path = config.option.durations_path + self._suite_num_tests = len(items) + if any((splits, group)): if not all((splits, group)): + self._group_num_tests = self._suite_num_tests return if not os.path.isfile(durations_report_path): + self._group_num_tests = self._suite_num_tests return if store_durations: # Don't split if we are storing durations + self._group_num_tests = self._suite_num_tests return - total_tests_count = len(items) + + self._group_num_tests = self._suite_num_tests + if splits and group: with open(durations_report_path) as f: stored_durations = OrderedDict(json.load(f)) @@ -66,17 +83,9 @@ def pytest_collection_modifyitems(self, session, config, items): start_idx, end_idx = _calculate_suite_start_and_end_idx( splits, group, items, stored_durations ) + self._group_num_tests = end_idx - start_idx items[:] = items[start_idx:end_idx] - terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") - terminal_writer = create_terminal_writer(config) - message = terminal_writer.markup( - " Running group {}/{} ({}/{} tests)\n".format( - group, splits, len(items), total_tests_count - ) - ) - terminal_reporter.write(message) - def pytest_sessionfinish(self, session, exitstatus): if session.config.option.store_durations: report_path = session.config.option.durations_path diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 9f3eee6..e11db28 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -214,5 +214,14 @@ def test_it_adapts_splits_based_on_new_and_deleted_tests( ] +class TestHasExpectedOutput: + def test_prints_splitting_summary(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "1", "--group", 1) + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "Running group 1/1 (10/10) tests" in outerr.out + + def _passed_test_names(result): return [passed.nodeid.split("::")[-1] for passed in result.listoutcomes()[0]] From 4d20239b9d8de0aa290fbd73b69bf0ffff9417a2 Mon Sep 17 00:00:00 2001 From: Maurits Kroese Date: Tue, 18 May 2021 22:02:00 +0200 Subject: [PATCH 3/9] Added more messaging in case tests are not split --- src/pytest_split/plugin.py | 48 +++++++++++++++++++++++++++----------- tests/test_plugin.py | 43 +++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 7557acd..1e8e9ec 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -1,12 +1,16 @@ import json import os -from collections import defaultdict, OrderedDict +from collections import defaultdict, OrderedDict, namedtuple +from typing import List from _pytest.config import create_terminal_writer # Ugly hacks for freezegun compatibility: https://github.com/spulec/freezegun/issues/286 STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD = 60 * 10 # seconds +TestGroup = namedtuple("TestGroup", "index, num_tests") +TestSuite = namedtuple("TestSuite", "splits, num_tests") + def pytest_addoption(parser): group = parser.getgroup( @@ -45,37 +49,51 @@ def pytest_addoption(parser): class SplitPlugin: def __init__(self): - self._suite_num_tests: int - self._group_num_tests: int + self._suite: TestSuite + self._group: TestGroup + self._messages: List[str] = [] def pytest_report_collectionfinish(self, config): - return ( - f"Running group {config.option.group}/{config.option.splits}" - f" ({self._group_num_tests}/{self._suite_num_tests}) tests" + lines = [] + if self._messages: + lines += self._messages + lines.append( + f"Running group {self._group.index}/{self._suite.splits}" + f" ({self._group.num_tests}/{self._suite.num_tests}) tests" ) + prefix = "[pytest-split]" + lines = [f"{prefix} {m}" for m in lines] + + return lines + def pytest_collection_modifyitems(self, session, config, items): splits = config.option.splits group = config.option.group store_durations = config.option.store_durations durations_report_path = config.option.durations_path - self._suite_num_tests = len(items) + self._suite = TestSuite(splits if splits is not None else 1, len(items)) if any((splits, group)): - if not all((splits, group)): - self._group_num_tests = self._suite_num_tests + if not group: + self._messages.append("Not splitting tests because the `group` argument is missing") + self._group = TestGroup(1, self._suite.num_tests) + return + if not splits: + self._messages.append("Not splitting tests because the `splits` argument is missing") + self._group = TestGroup(1, self._suite.num_tests) return if not os.path.isfile(durations_report_path): - self._group_num_tests = self._suite_num_tests + self._messages.append("Not splitting tests because the durations_report is missing") + self._group = TestGroup(1, self._suite.num_tests) return if store_durations: # Don't split if we are storing durations - self._group_num_tests = self._suite_num_tests + self._messages.append("Not splitting tests because we are storing durations") + self._group = TestGroup(1, self._suite.num_tests) return - self._group_num_tests = self._suite_num_tests - if splits and group: with open(durations_report_path) as f: stored_durations = OrderedDict(json.load(f)) @@ -83,8 +101,10 @@ def pytest_collection_modifyitems(self, session, config, items): start_idx, end_idx = _calculate_suite_start_and_end_idx( splits, group, items, stored_durations ) - self._group_num_tests = end_idx - start_idx + self._group = TestGroup(group, self._suite.num_tests) items[:] = items[start_idx:end_idx] + else: + self._group = TestGroup(1, self._suite.num_tests) def pytest_sessionfinish(self, session, exitstatus): if session.config.option.store_durations: diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e11db28..657e1b6 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -220,7 +220,48 @@ def test_prints_splitting_summary(self, example_suite, capsys): assert result.ret == 0 outerr = capsys.readouterr() - assert "Running group 1/1 (10/10) tests" in outerr.out + assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out + + def test_prints_splitting_summary_when_group_missing(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "1") + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "[pytest-split] Not splitting tests because the `group` argument is missing" in outerr.out + + def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys): + result = example_suite.inline_run("--group", "1") + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "[pytest-split] Not splitting tests because the `splits` argument is missing" in outerr.out + + def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys): + result = example_suite.inline_run("--splits", 1, "--group", "1") + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "[pytest-split] Not splitting tests because the durations_report is missing" in outerr.out + + def test_prints_splitting_summary_when_durations_missing(self, example_suite, capsys, durations_path): + test_name = "test_prints_splitting_summary_when_durations_missing" + with open(durations_path, "w") as f: + json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) + + result = example_suite.inline_run( + "--splits", 1, "--group", "1", "--durations-path", durations_path, "--store-durations" + ) + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "[pytest-split] Not splitting tests because we are storing durations" in outerr.out + + def test_prints_splitting_summary_when_no_pytest_split_arguments(self, example_suite, capsys): + result = example_suite.inline_run() + assert result.ret == 0 + + outerr = capsys.readouterr() + assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out def _passed_test_names(result): From e5724dc5df7861e2da426053935b0a2455830b3e Mon Sep 17 00:00:00 2001 From: mbk Date: Sat, 5 Jun 2021 14:49:36 +0200 Subject: [PATCH 4/9] Raise UsageError when required arguments are missing or invalid This commit adds two pieces of functionality: 1. verify that neither OR both splits and group are passed 2. verify that the group argument is within valid range --- src/pytest_split/plugin.py | 30 +++++++++++++++------ tests/test_plugin.py | 55 +++++++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 1e8e9ec..12cba28 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -1,4 +1,5 @@ import json +import pytest import os from collections import defaultdict, OrderedDict, namedtuple from typing import List @@ -47,6 +48,27 @@ def pytest_addoption(parser): ) +@pytest.mark.tryfirst +def pytest_cmdline_main(config): + group = config.getoption("group") + splits = config.getoption("splits") + + if splits is None and group is None: + return + + if splits and group is None: + raise pytest.UsageError("argument `--group` is required") + + if group and splits is None: + raise pytest.UsageError("argument `--splits` is required") + + if splits < 1: + raise pytest.UsageError("argument `--splits` must be >= 1") + + if group < 1 or group > splits: + raise pytest.UsageError(f"argument `--group` must be >= 1 and <= {splits}") + + class SplitPlugin: def __init__(self): self._suite: TestSuite @@ -76,14 +98,6 @@ def pytest_collection_modifyitems(self, session, config, items): self._suite = TestSuite(splits if splits is not None else 1, len(items)) if any((splits, group)): - if not group: - self._messages.append("Not splitting tests because the `group` argument is missing") - self._group = TestGroup(1, self._suite.num_tests) - return - if not splits: - self._messages.append("Not splitting tests because the `splits` argument is missing") - self._group = TestGroup(1, self._suite.num_tests) - return if not os.path.isfile(durations_report_path): self._messages.append("Not splitting tests because the durations_report is missing") self._group = TestGroup(1, self._suite.num_tests) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 657e1b6..e6e2141 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -155,16 +155,6 @@ def test_it_does_not_split_with_invalid_args(self, example_suite, durations_path with open(durations_path, "w") as f: json.dump(durations, f) - result = example_suite.inline_run( - "--splits", "2", "--durations-path", durations_path - ) # no --group - result.assertoutcome(passed=10) - - result = example_suite.inline_run( - "--group", "2", "--durations-path", durations_path - ) # no --splits - result.assertoutcome(passed=10) - result = example_suite.inline_run( "--splits", "2", "--group", "1" ) # no durations report in default location @@ -214,27 +204,50 @@ def test_it_adapts_splits_based_on_new_and_deleted_tests( ] -class TestHasExpectedOutput: - def test_prints_splitting_summary(self, example_suite, capsys): - result = example_suite.inline_run("--splits", "1", "--group", 1) - assert result.ret == 0 +class TestRaisesUsageErrors: + def test_returns_nonzero_when_group_but_not_splits(self, example_suite, capsys): + result = example_suite.inline_run("--group", "1") + assert result.ret == 4 outerr = capsys.readouterr() - assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out + assert "argument `--splits` is required" in outerr.err - def test_prints_splitting_summary_when_group_missing(self, example_suite, capsys): + def test_returns_nonzero_when_splits_but_not_group(self, example_suite, capsys): result = example_suite.inline_run("--splits", "1") - assert result.ret == 0 + assert result.ret == 4 outerr = capsys.readouterr() - assert "[pytest-split] Not splitting tests because the `group` argument is missing" in outerr.out + assert "argument `--group` is required" in outerr.err - def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys): - result = example_suite.inline_run("--group", "1") + def test_returns_nonzero_when_group_below_one(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "3", "--group", 0) + assert result.ret == 4 + + outerr = capsys.readouterr() + assert "argument `--group` must be >= 1 and <= 3" in outerr.err + + def test_returns_nonzero_when_group_larger_than_splits(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "3", "--group", 4) + assert result.ret == 4 + + outerr = capsys.readouterr() + assert "argument `--group` must be >= 1 and <= 3" in outerr.err + + def test_returns_nonzero_when_splits_below_one(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "0", "--group", "1") + assert result.ret == 4 + + outerr = capsys.readouterr() + assert "argument `--splits` must be >= 1" in outerr.err + + +class TestHasExpectedOutput: + def test_prints_splitting_summary(self, example_suite, capsys): + result = example_suite.inline_run("--splits", "1", "--group", 1) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Not splitting tests because the `splits` argument is missing" in outerr.out + assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys): result = example_suite.inline_run("--splits", 1, "--group", "1") From b4c70aebd62beff86333a176ef6e2617633bb208 Mon Sep 17 00:00:00 2001 From: mbk Date: Sat, 5 Jun 2021 15:24:05 +0200 Subject: [PATCH 5/9] Dont split and print messages unless explicitly invoked --- src/pytest_split/plugin.py | 7 +++++++ tests/test_plugin.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 12cba28..e7520ee 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -76,6 +76,9 @@ def __init__(self): self._messages: List[str] = [] def pytest_report_collectionfinish(self, config): + if not hasattr(self, "_suite"): + return + lines = [] if self._messages: lines += self._messages @@ -95,6 +98,10 @@ def pytest_collection_modifyitems(self, session, config, items): store_durations = config.option.store_durations durations_report_path = config.option.durations_path + if not any([group, splits, store_durations]): + # don't split unless explicitly requested + return + self._suite = TestSuite(splits if splits is not None else 1, len(items)) if any((splits, group)): diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e6e2141..b494e6b 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -269,12 +269,12 @@ def test_prints_splitting_summary_when_durations_missing(self, example_suite, ca outerr = capsys.readouterr() assert "[pytest-split] Not splitting tests because we are storing durations" in outerr.out - def test_prints_splitting_summary_when_no_pytest_split_arguments(self, example_suite, capsys): + def test_does_not_print_splitting_summary_when_no_pytest_split_arguments(self, example_suite, capsys): result = example_suite.inline_run() assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out + assert "[pytest-split]" not in outerr.out def _passed_test_names(result): From e967ab02947d8a9e3f4603bbb26a4b56e40b9b85 Mon Sep 17 00:00:00 2001 From: mbk Date: Sat, 5 Jun 2021 16:08:28 +0200 Subject: [PATCH 6/9] Simplified splitting logic that could be simplified after previous changes --- src/pytest_split/plugin.py | 52 +++++++++++++++++--------------------- tests/test_plugin.py | 14 ++++++---- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index e7520ee..6ed06e7 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -76,16 +76,15 @@ def __init__(self): self._messages: List[str] = [] def pytest_report_collectionfinish(self, config): - if not hasattr(self, "_suite"): - return - lines = [] if self._messages: lines += self._messages - lines.append( - f"Running group {self._group.index}/{self._suite.splits}" - f" ({self._group.num_tests}/{self._suite.num_tests}) tests" - ) + + if hasattr(self, "_suite"): + lines.append( + f"Running group {self._group.index}/{self._suite.splits}" + f" ({self._group.num_tests}/{self._suite.num_tests}) tests" + ) prefix = "[pytest-split]" lines = [f"{prefix} {m}" for m in lines] @@ -98,34 +97,29 @@ def pytest_collection_modifyitems(self, session, config, items): store_durations = config.option.store_durations durations_report_path = config.option.durations_path - if not any([group, splits, store_durations]): + if store_durations: + if any((group, splits)): + self._messages.append("Not splitting tests because we are storing durations") + return + + if not group and not splits: # don't split unless explicitly requested return - self._suite = TestSuite(splits if splits is not None else 1, len(items)) + if not os.path.isfile(durations_report_path): + self._messages.append("Not splitting tests because the durations_report is missing") + return - if any((splits, group)): - if not os.path.isfile(durations_report_path): - self._messages.append("Not splitting tests because the durations_report is missing") - self._group = TestGroup(1, self._suite.num_tests) - return - if store_durations: - # Don't split if we are storing durations - self._messages.append("Not splitting tests because we are storing durations") - self._group = TestGroup(1, self._suite.num_tests) - return + with open(durations_report_path) as f: + stored_durations = OrderedDict(json.load(f)) - if splits and group: - with open(durations_report_path) as f: - stored_durations = OrderedDict(json.load(f)) + start_idx, end_idx = _calculate_suite_start_and_end_idx( + splits, group, items, stored_durations + ) - start_idx, end_idx = _calculate_suite_start_and_end_idx( - splits, group, items, stored_durations - ) - self._group = TestGroup(group, self._suite.num_tests) - items[:] = items[start_idx:end_idx] - else: - self._group = TestGroup(1, self._suite.num_tests) + self._suite = TestSuite(splits, len(items)) + self._group = TestGroup(group, end_idx-start_idx) + items[:] = items[start_idx:end_idx] def pytest_sessionfinish(self, session, exitstatus): if session.config.option.store_durations: diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b494e6b..b706cb9 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -242,19 +242,23 @@ def test_returns_nonzero_when_splits_below_one(self, example_suite, capsys): class TestHasExpectedOutput: - def test_prints_splitting_summary(self, example_suite, capsys): + def test_does_not_print_splitting_summary_when_durations_missing(self, example_suite, capsys): result = example_suite.inline_run("--splits", "1", "--group", 1) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out + assert "[pytest-split] Not splitting tests because the durations_report is missing" in outerr.out + assert "[pytest-split] Running group" not in outerr.out - def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys): - result = example_suite.inline_run("--splits", 1, "--group", "1") + def test_prints_splitting_summary_when_durations_present(self, example_suite, capsys, durations_path): + test_name = "test_prints_splitting_summary_when_durations_present" + with open(durations_path, "w") as f: + json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) + result = example_suite.inline_run("--splits", "1", "--group", 1) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Not splitting tests because the durations_report is missing" in outerr.out + assert "[pytest-split] Running group 1/1 (10/10) tests" def test_prints_splitting_summary_when_durations_missing(self, example_suite, capsys, durations_path): test_name = "test_prints_splitting_summary_when_durations_missing" From a2c31690064f7f7da4003940d1ea032db7db1373 Mon Sep 17 00:00:00 2001 From: mbk Date: Sat, 5 Jun 2021 16:28:36 +0200 Subject: [PATCH 7/9] Added extra typing information --- src/pytest_split/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 0709060..3c435e7 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -57,7 +57,7 @@ def pytest_addoption(parser: "Parser") -> None: @pytest.mark.tryfirst -def pytest_cmdline_main(config): +def pytest_cmdline_main(config: "Config") -> None: group = config.getoption("group") splits = config.getoption("splits") @@ -83,7 +83,7 @@ def __init__(self): self._group: TestGroup self._messages: List[str] = [] - def pytest_report_collectionfinish(self, config): + def pytest_report_collectionfinish(self, config: "Config") -> "List[str]": lines = [] if self._messages: lines += self._messages From a6306688e6b97c7673a227442b9d82563ad452cc Mon Sep 17 00:00:00 2001 From: mbk Date: Mon, 7 Jun 2021 08:43:26 +0200 Subject: [PATCH 8/9] Fix pre-commit violations --- src/pytest_split/plugin.py | 16 ++++++++++----- tests/test_plugin.py | 40 +++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/pytest_split/plugin.py b/src/pytest_split/plugin.py index 3c435e7..3ac4ff0 100644 --- a/src/pytest_split/plugin.py +++ b/src/pytest_split/plugin.py @@ -81,7 +81,7 @@ class SplitPlugin: def __init__(self): self._suite: TestSuite self._group: TestGroup - self._messages: List[str] = [] + self._messages: "List[str]" = [] def pytest_report_collectionfinish(self, config: "Config") -> "List[str]": lines = [] @@ -99,7 +99,9 @@ def pytest_report_collectionfinish(self, config: "Config") -> "List[str]": return lines - def pytest_collection_modifyitems(self, config: "Config", items: "List[nodes.Item]") -> None: + def pytest_collection_modifyitems( + self, config: "Config", items: "List[nodes.Item]" + ) -> None: splits = config.option.splits group = config.option.group store_durations = config.option.store_durations @@ -107,7 +109,9 @@ def pytest_collection_modifyitems(self, config: "Config", items: "List[nodes.Ite if store_durations: if any((group, splits)): - self._messages.append("Not splitting tests because we are storing durations") + self._messages.append( + "Not splitting tests because we are storing durations" + ) return None if not group and not splits: @@ -115,7 +119,9 @@ def pytest_collection_modifyitems(self, config: "Config", items: "List[nodes.Ite return None if not os.path.isfile(durations_report_path): - self._messages.append("Not splitting tests because the durations_report is missing") + self._messages.append( + "Not splitting tests because the durations_report is missing" + ) return None with open(durations_report_path) as f: @@ -126,7 +132,7 @@ def pytest_collection_modifyitems(self, config: "Config", items: "List[nodes.Ite ) self._suite = TestSuite(splits, len(items)) - self._group = TestGroup(group, end_idx-start_idx) + self._group = TestGroup(group, end_idx - start_idx) items[:] = items[start_idx:end_idx] diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b706cb9..b9c04dc 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -242,38 +242,60 @@ def test_returns_nonzero_when_splits_below_one(self, example_suite, capsys): class TestHasExpectedOutput: - def test_does_not_print_splitting_summary_when_durations_missing(self, example_suite, capsys): + def test_does_not_print_splitting_summary_when_durations_missing( + self, example_suite, capsys + ): result = example_suite.inline_run("--splits", "1", "--group", 1) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Not splitting tests because the durations_report is missing" in outerr.out + assert ( + "[pytest-split] Not splitting tests because the durations_report is missing" + in outerr.out + ) assert "[pytest-split] Running group" not in outerr.out - def test_prints_splitting_summary_when_durations_present(self, example_suite, capsys, durations_path): + def test_prints_splitting_summary_when_durations_present( + self, example_suite, capsys, durations_path + ): test_name = "test_prints_splitting_summary_when_durations_present" with open(durations_path, "w") as f: json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) - result = example_suite.inline_run("--splits", "1", "--group", 1) + result = example_suite.inline_run( + "--splits", "1", "--group", 1, "--durations-path", durations_path + ) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Running group 1/1 (10/10) tests" + assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out - def test_prints_splitting_summary_when_durations_missing(self, example_suite, capsys, durations_path): + def test_prints_splitting_summary_when_durations_missing( + self, example_suite, capsys, durations_path + ): test_name = "test_prints_splitting_summary_when_durations_missing" with open(durations_path, "w") as f: json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) result = example_suite.inline_run( - "--splits", 1, "--group", "1", "--durations-path", durations_path, "--store-durations" + "--splits", + 1, + "--group", + "1", + "--durations-path", + durations_path, + "--store-durations", ) assert result.ret == 0 outerr = capsys.readouterr() - assert "[pytest-split] Not splitting tests because we are storing durations" in outerr.out + assert ( + "[pytest-split] Not splitting tests because we are storing durations" + in outerr.out + ) - def test_does_not_print_splitting_summary_when_no_pytest_split_arguments(self, example_suite, capsys): + def test_does_not_print_splitting_summary_when_no_pytest_split_arguments( + self, example_suite, capsys + ): result = example_suite.inline_run() assert result.ret == 0 From 38064b04e871a57f7c218fee126b6657319ed97d Mon Sep 17 00:00:00 2001 From: mbk Date: Tue, 8 Jun 2021 09:59:11 +0200 Subject: [PATCH 9/9] Fix tests: pass integers as strings in tests --- tests/test_plugin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b9c04dc..4a769ec 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -220,14 +220,14 @@ def test_returns_nonzero_when_splits_but_not_group(self, example_suite, capsys): assert "argument `--group` is required" in outerr.err def test_returns_nonzero_when_group_below_one(self, example_suite, capsys): - result = example_suite.inline_run("--splits", "3", "--group", 0) + result = example_suite.inline_run("--splits", "3", "--group", "0") assert result.ret == 4 outerr = capsys.readouterr() assert "argument `--group` must be >= 1 and <= 3" in outerr.err def test_returns_nonzero_when_group_larger_than_splits(self, example_suite, capsys): - result = example_suite.inline_run("--splits", "3", "--group", 4) + result = example_suite.inline_run("--splits", "3", "--group", "4") assert result.ret == 4 outerr = capsys.readouterr() @@ -245,7 +245,7 @@ class TestHasExpectedOutput: def test_does_not_print_splitting_summary_when_durations_missing( self, example_suite, capsys ): - result = example_suite.inline_run("--splits", "1", "--group", 1) + result = example_suite.inline_run("--splits", "1", "--group", "1") assert result.ret == 0 outerr = capsys.readouterr() @@ -262,23 +262,23 @@ def test_prints_splitting_summary_when_durations_present( with open(durations_path, "w") as f: json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) result = example_suite.inline_run( - "--splits", "1", "--group", 1, "--durations-path", durations_path + "--splits", "1", "--group", "1", "--durations-path", durations_path ) assert result.ret == 0 outerr = capsys.readouterr() assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out - def test_prints_splitting_summary_when_durations_missing( + def test_prints_splitting_summary_when_storing_durations( self, example_suite, capsys, durations_path ): - test_name = "test_prints_splitting_summary_when_durations_missing" + test_name = "test_prints_splitting_summary_when_storing_durations" with open(durations_path, "w") as f: json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f) result = example_suite.inline_run( "--splits", - 1, + "1", "--group", "1", "--durations-path",