Update messaging in case splitting is skipped and check for UsageErrors more aggressively

src/pytest_split/plugin.py

@@ -1,12 +1,16 @@
 import json
 import os
-from collections import defaultdict, OrderedDict
+from collections import defaultdict, OrderedDict, namedtuple
+from typing import List
 
 from _pytest.config import create_terminal_writer
 
 # Ugly hacks for freezegun compatibility: https://github.com/spulec/freezegun/issues/286
 STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD = 60 * 10  # seconds
 
+TestGroup = namedtuple("TestGroup", "index, num_tests")
+TestSuite = namedtuple("TestSuite", "splits, num_tests")
+
 
 def pytest_addoption(parser):
     group = parser.getgroup(
@@ -43,69 +47,98 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_collection_modifyitems(session, config, items):
-    splits = config.option.splits
-    group = config.option.group
-    store_durations = config.option.store_durations
-    durations_report_path = config.option.durations_path
-
-    if any((splits, group)):
-        if not all((splits, group)):
-            return
-        if not os.path.isfile(durations_report_path):
-            return
-        if store_durations:
-            # Don't split if we are storing durations
-            return
-    total_tests_count = len(items)
-    if splits and group:
-        with open(durations_report_path) as f:
-            stored_durations = OrderedDict(json.load(f))
-
-        start_idx, end_idx = _calculate_suite_start_and_end_idx(
-            splits, group, items, stored_durations
+class SplitPlugin:
+    def __init__(self):
+        self._suite: TestSuite
+        self._group: TestGroup
+        self._messages: List[str] = []
+
+    def pytest_report_collectionfinish(self, config):
+        lines = []
+        if self._messages:
+            lines += self._messages
+        lines.append(
+            f"Running group {self._group.index}/{self._suite.splits}"
+            f" ({self._group.num_tests}/{self._suite.num_tests}) tests"
         )
-        items[:] = items[start_idx:end_idx]
 
-        terminal_reporter = config.pluginmanager.get_plugin("terminalreporter")
-        terminal_writer = create_terminal_writer(config)
-        message = terminal_writer.markup(
-            " Running group {}/{} ({}/{} tests)\n".format(
-                group, splits, len(items), total_tests_count
+        prefix = "[pytest-split]"
+        lines = [f"{prefix} {m}" for m in lines]
+
+        return lines
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        splits = config.option.splits
+        group = config.option.group
+        store_durations = config.option.store_durations
+        durations_report_path = config.option.durations_path
+
+        self._suite = TestSuite(splits if splits is not None else 1, len(items))
+
+        if any((splits, group)):
+            if not group:
+                self._messages.append("Not splitting tests because the `group` argument is missing")
+                self._group = TestGroup(1, self._suite.num_tests)
+                return
+            if not splits:
+                self._messages.append("Not splitting tests because the `splits` argument is missing")
+                self._group = TestGroup(1, self._suite.num_tests)
+                return
+            if not os.path.isfile(durations_report_path):
+                self._messages.append("Not splitting tests because the durations_report is missing")
+                self._group = TestGroup(1, self._suite.num_tests)
+                return
+            if store_durations:
+                # Don't split if we are storing durations
+                self._messages.append("Not splitting tests because we are storing durations")
+                self._group = TestGroup(1, self._suite.num_tests)
+                return
+
+        if splits and group:
+            with open(durations_report_path) as f:
+                stored_durations = OrderedDict(json.load(f))
+
+            start_idx, end_idx = _calculate_suite_start_and_end_idx(
+                splits, group, items, stored_durations
             )
-        )
-        terminal_reporter.write(message)
-
-
-def pytest_sessionfinish(session, exitstatus):
-    if session.config.option.store_durations:
-        report_path = session.config.option.durations_path
-        terminal_reporter = session.config.pluginmanager.get_plugin("terminalreporter")
-        durations = defaultdict(float)
-        for test_reports in terminal_reporter.stats.values():
-            for test_report in test_reports:
-                if hasattr(test_report, "duration"):
-                    stage = getattr(test_report, "when", "")
-                    duration = test_report.duration
-                    # These ifs be removed after this is solved: https://github.com/spulec/freezegun/issues/286
-                    if duration < 0:
-                        continue
-                    if (
-                        stage in ("teardown", "setup")
-                        and duration > STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD
-                    ):
-                        # Ignore not legit teardown durations
-                        continue
-                    durations[test_report.nodeid] += test_report.duration
-
-        with open(report_path, "w") as f:
-            f.write(json.dumps(list(durations.items()), indent=2))
-
-        terminal_writer = create_terminal_writer(session.config)
-        message = terminal_writer.markup(
-            " Stored test durations in {}\n".format(report_path)
-        )
-        terminal_reporter.write(message)
+            self._group = TestGroup(group, self._suite.num_tests)
+            items[:] = items[start_idx:end_idx]
+        else:
+            self._group = TestGroup(1, self._suite.num_tests)
+
+    def pytest_sessionfinish(self, session, exitstatus):
+        if session.config.option.store_durations:
+            report_path = session.config.option.durations_path
+            terminal_reporter = session.config.pluginmanager.get_plugin("terminalreporter")
+            durations = defaultdict(float)
+            for test_reports in terminal_reporter.stats.values():
+                for test_report in test_reports:
+                    if hasattr(test_report, "duration"):
+                        stage = getattr(test_report, "when", "")
+                        duration = test_report.duration
+                        # These ifs be removed after this is solved: https://github.com/spulec/freezegun/issues/286
+                        if duration < 0:
+                            continue
+                        if (
+                            stage in ("teardown", "setup")
+                            and duration > STORE_DURATIONS_SETUP_AND_TEARDOWN_THRESHOLD
+                        ):
+                            # Ignore not legit teardown durations
+                            continue
+                        durations[test_report.nodeid] += test_report.duration
+
+            with open(report_path, "w") as f:
+                f.write(json.dumps(list(durations.items()), indent=2))
+
+            terminal_writer = create_terminal_writer(session.config)
+            message = terminal_writer.markup(
+                " Stored test durations in {}\n".format(report_path)
+            )
+            terminal_reporter.write(message)
+
+
+def pytest_configure(config):
+    config.pluginmanager.register(SplitPlugin())
 
 
 def _calculate_suite_start_and_end_idx(splits, group, items, stored_durations):

tests/test_plugin.py

@@ -214,5 +214,55 @@ def test_it_adapts_splits_based_on_new_and_deleted_tests(
         ]
 
 
+class TestHasExpectedOutput:
+    def test_prints_splitting_summary(self, example_suite, capsys):
+        result = example_suite.inline_run("--splits", "1", "--group", 1)
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out
+
+    def test_prints_splitting_summary_when_group_missing(self, example_suite, capsys):
+        result = example_suite.inline_run("--splits", "1")
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Not splitting tests because the `group` argument is missing" in outerr.out
+
+    def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys):
+        result = example_suite.inline_run("--group", "1")
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Not splitting tests because the `splits` argument is missing" in outerr.out
+
+    def test_prints_splitting_summary_when_splits_missing(self, example_suite, capsys):
+        result = example_suite.inline_run("--splits", 1, "--group", "1")
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Not splitting tests because the durations_report is missing" in outerr.out
+
+    def test_prints_splitting_summary_when_durations_missing(self, example_suite, capsys, durations_path):
+        test_name = "test_prints_splitting_summary_when_durations_missing"
+        with open(durations_path, "w") as f:
+            json.dump([[f"{test_name}0/{test_name}.py::test_1", 0.5]], f)
+
+        result = example_suite.inline_run(
+            "--splits", 1, "--group", "1", "--durations-path", durations_path, "--store-durations"
+        )
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Not splitting tests because we are storing durations" in outerr.out
+
+    def test_prints_splitting_summary_when_no_pytest_split_arguments(self, example_suite, capsys):
+        result = example_suite.inline_run()
+        assert result.ret == 0
+
+        outerr = capsys.readouterr()
+        assert "[pytest-split] Running group 1/1 (10/10) tests" in outerr.out
+
+
 def _passed_test_names(result):
     return [passed.nodeid.split("::")[-1] for passed in result.listoutcomes()[0]]