Merge pull request #3 from NellyWhads/test_breakdown

Add support for test breakdown per worker

Author: mikicz

README.md

@@ -12,33 +12,74 @@ $ pip install pytest-xdist-worker-stats
 
 All that is needed is to have xdist installed & enabled, and to run tests in multiple workers.
 
-## Example output
+### Default mode
+
+```shell
+pytest {all_your_options}
+```
+
+```text
+============================= test session starts ==============================
+platform linux -- Python 3.10.7, pytest-8.1.1, pluggy-1.4.0
+plugins: xdist-worker-stats-0.2.0, xdist-3.5.0
+created: 2/2 workers
+2 workers [4 items]
+
+....                                                                     [100%]
+============================== Worker statistics ===============================
+worker gw0  :    2 tests       0.00s runtime
+worker gw1  :    2 tests       0.00s runtime
+
+Tests   : min        2, max        2, average 2.0
+Runtime : min    0.00s, max    0.00s, average 0.00s
+============================== 4 passed in 1.82s ===============================
+```
+
+### Summary mode
+
+```shell
+pytest {all_your_options} --no-xdist-runtimes
+```
+
+```text
+============================= test session starts ==============================
+platform linux -- Python 3.10.7, pytest-8.1.1, pluggy-1.4.0
+plugins: xdist-worker-stats-0.2.0, xdist-3.5.0
+created: 2/2 workers
+2 workers [4 items]
+
+....                                                                     [100%]
+============================== Worker statistics ===============================
+Tests   : min        2, max        2, average 2.0
+Runtime : min    0.00s, max    0.00s, average 0.00s
+============================== 4 passed in 1.82s ===============================
+```
+
+### Breakdown mode
+
+```shell
+pytest {all_your_options} --xdist-breakdown
+```
 
 ```text
-platform linux -- Python 3.10.11, pytest-7.3.2, pluggy-1.0.0
-plugins: xdist-3.3.1, xdist-worker-stats-0.1.0
-12 workers [359 items]
-.............................................................................................. [ 25%]
-.............................................................................................. [ 52%]
-.............................................................................................. [ 78%]
-.............................................................................                  [100%]
-========================================= Worker statistics ==========================================
-worker gw0  :   15 tests      12.25s runtime
-worker gw1  :   14 tests      12.00s runtime
-worker gw2  :   27 tests      11.66s runtime
-worker gw3  :   13 tests      12.08s runtime
-worker gw4  :   14 tests      12.59s runtime
-worker gw5  :   27 tests      12.13s runtime
-worker gw6  :   18 tests      12.22s runtime
-worker gw7  :   78 tests      12.04s runtime
-worker gw8  :   21 tests      12.01s runtime
-worker gw9  :   59 tests      12.36s runtime
-worker gw10 :   20 tests      11.79s runtime
-worker gw11 :   53 tests      12.09s runtime
-
-Tests   : min       13, max       78, average 29.9
-Runtime : min   11.66s, max   12.59s, average 12.10s
-======================================== 359 passed in 21.52s ========================================
+============================= test session starts ==============================
+platform linux -- Python 3.10.7, pytest-8.1.1, pluggy-1.4.0
+plugins: xdist-worker-stats-0.2.0, xdist-3.5.0
+created: 2/2 workers
+2 workers [4 items]
+
+....                                                                     [100%]
+============================== Worker statistics ===============================
+worker gw0  :    2 tests       0.00s runtime
+    test_plugin.py::test_bar[1]
+    test_plugin.py::test_foo
+worker gw1  :    2 tests       0.00s runtime
+    test_plugin.py::test_bar[2]
+    test_plugin.py::test_bar[3]
+
+Tests   : min        2, max        2, average 2.0
+Runtime : min    0.00s, max    0.00s, average 0.00s
+============================== 4 passed in 1.82s ===============================
 ```
 
 ## Development

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytest-xdist-worker-stats"
-version = "0.1.6"
+version = "0.2.0"
 description = "A pytest plugin to list worker statistics after a xdist run."
 authors = ["Mikuláš Poul <[email protected]>"]
 license = "MIT"
@@ -29,8 +29,8 @@ pytest-xdist-worker-stats = "pytest_xdist_worker_stats"
 
 [tool.poetry.dependencies]
 python = ">=3.8"
-pytest = ">7.3.2"
-pytest-xdist = "^3.3"
+pytest = ">=7.0.0"
+pytest-xdist = ">=3"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.9.1"

pytest_xdist_worker_stats/__init__.py

@@ -1,3 +1,3 @@
-from .options import pytest_configure  # noqa: F401
+from pytest_xdist_worker_stats.options import pytest_addoption, pytest_configure  # noqa: F401
 
-__version__ = "0.1.6"
+__version__ = "0.2.0"

pytest_xdist_worker_stats/options.py

@@ -1,24 +1,30 @@
-from typing import TYPE_CHECKING, NoReturn
+import pytest
 
-if TYPE_CHECKING:
-    from _pytest.config import Config, PytestPluginManager
-    from _pytest.config.argparsing import Parser
 
+def pytest_addoption(parser: pytest.Parser):
+    from pytest_xdist_worker_stats.plugin import (
+        ARGPARSE_PARSER_GROUP,
+        ARGPARSE_REPORT_TEST_BREAKDOWN_OPTION_NAME,
+        ARGPARSE_REPORT_WORKER_RUNTIMES_OPTION_NAME,
+    )
 
-def pytest_addoption(parser: "Parser", pluginmanager: "PytestPluginManager") -> NoReturn:
-    group = parser.getgroup("pytest-unused-fixtures")
-    group.addoption("--unused-fixtures", action="store_true", default=False, help="Try to identify unused fixtures.")
+    group = parser.getgroup(ARGPARSE_PARSER_GROUP)
+    group.addoption(
+        "--no-xdist-runtimes",
+        action="store_false",
+        default=True,
+        dest=ARGPARSE_REPORT_WORKER_RUNTIMES_OPTION_NAME,
+        help="Do not report runtimes per 'xdist' worker.",
+    )
     group.addoption(
-        "--unused-fixtures-ignore-path",
-        metavar="PATH",
-        type=str,
-        default=None,
-        action="append",
-        help="Ignore fixtures in PATHs from unused fixtures report.",
+        "--xdist-breakdown",
+        action="store_true",
+        dest=ARGPARSE_REPORT_TEST_BREAKDOWN_OPTION_NAME,
+        help="Display test breakdown per 'xdist' worker.",
     )
 
 
-def pytest_configure(config: "Config") -> NoReturn:
+def pytest_configure(config: pytest.Config):
     pluginmanager = config.pluginmanager
     if pluginmanager.hasplugin("xdist"):
         from pytest_xdist_worker_stats.plugin import XdistWorkerStatsPlugin

pytest_xdist_worker_stats/plugin.py

@@ -3,11 +3,16 @@
 from typing import NamedTuple
 
 import pytest
+from _pytest.terminal import TerminalReporter
+
+ARGPARSE_PARSER_GROUP = "pytest-xdist-worker-stats"
+ARGPARSE_REPORT_WORKER_RUNTIMES_OPTION_NAME = "pytest_xdist_worker_stats_report_worker_runtimes"
+ARGPARSE_REPORT_TEST_BREAKDOWN_OPTION_NAME = "pytest_xdist_worker_stats_report_test_breakdown"
 
 SHARED_WORKER_INFO = "worker_info"
 
 
-class RunStatistics(NamedTuple):
+class RuntimeStats(NamedTuple):
     mininum_tests: int
     maximum_tests: int
     average_tests: float
@@ -17,10 +22,12 @@ class RunStatistics(NamedTuple):
 
 
 class XdistWorkerStatsPlugin:
-    def __init__(self, config):
+    def __init__(self, config: pytest.Config):
         self.config = config
         self.test_stats = {}
-        self.worker_test_times = {}
+        self.worker_stats = {}
+        self.report_worker_runtimes = config.getoption(ARGPARSE_REPORT_WORKER_RUNTIMES_OPTION_NAME, False)
+        self.report_test_breakdown = config.getoption(ARGPARSE_REPORT_TEST_BREAKDOWN_OPTION_NAME, False)
 
     def add(self, name):
         self.test_stats[name] = self.test_stats.get(name) or {}
@@ -32,52 +39,57 @@ def pytest_runtest_setup(self, item):
     @pytest.hookimpl(hookwrapper=True)
     def pytest_runtest_call(self, item):
         yield
-        end = time.time()
-        self.add(item.nodeid)["diff"] = end - self.add(item.nodeid)["start"]
-
-        if (worker := os.environ.get("PYTEST_XDIST_WORKER", "primary")) not in self.worker_test_times:
-            self.worker_test_times[worker] = []
-
-        self.worker_test_times[worker].append(self.add(item.nodeid)["diff"])
-
-    def get_statistics(self) -> RunStatistics:
-        workers = self.worker_test_times.keys()
-        tests = [len(self.worker_test_times[worker]) for worker in workers]
-        runtimes = [sum(self.worker_test_times[worker]) for worker in workers]
-
-        return RunStatistics(
-            mininum_tests=min(tests),
-            maximum_tests=max(tests),
-            average_tests=sum(tests) / len(tests),
-            mininum_runtime=min(runtimes),
-            maximum_runtime=max(runtimes),
-            average_runtime=sum(runtimes) / len(runtimes),
+        runtime = time.time() - self.add(item.nodeid)["start"]
+        self.add(item.nodeid)["runtime"] = runtime
+
+        if (worker := os.environ.get("PYTEST_XDIST_WORKER", "primary")) not in self.worker_stats:
+            self.worker_stats[worker] = {}
+
+        self.worker_stats[worker][item.nodeid] = runtime
+
+    def get_runtime_stats(self) -> RuntimeStats:
+        test_counts = [len(stats) for stats in self.worker_stats.values()]
+        test_runtimes = [sum(stats.values()) for stats in self.worker_stats.values()]
+
+        return RuntimeStats(
+            mininum_tests=min(test_counts),
+            maximum_tests=max(test_counts),
+            average_tests=sum(test_counts) / len(test_counts),
+            mininum_runtime=min(test_runtimes),
+            maximum_runtime=max(test_runtimes),
+            average_runtime=sum(test_runtimes) / len(test_runtimes),
         )
 
-    def pytest_terminal_summary(self, terminalreporter):
+    def pytest_terminal_summary(self, terminalreporter: TerminalReporter):
         """
         If there's multiple workers, report on number of tests and total runtime.
         """
         tr = terminalreporter
-        if self.worker_test_times and len(self.worker_test_times) > 1:
+        if self.worker_stats and len(self.worker_stats) > 1:
             tr._tw.sep("=", "Worker statistics", yellow=True)
-            workers = sorted(self.worker_test_times.keys(), key=lambda x: int(x.lstrip("gw")))
-            statistics = self.get_statistics()
-
-            for worker in workers:
-                worker_times = self.worker_test_times[worker]
-                tr._tw.line(f"worker {worker: <5}: {len(worker_times): >4} tests {sum(worker_times):10.2f}s runtime")
-
-            tr._tw.line("")
+            worker_columns = len(max(self.worker_stats.keys(), key=len)) + 2
+
+            if self.report_worker_runtimes:
+                for worker, stats in sorted(self.worker_stats.items()):
+                    runtimes = stats.values()
+                    tr._tw.line(
+                        f"worker {worker: <{worker_columns}}: {len(runtimes): >4} tests {sum(runtimes):10.2f}s runtime"
+                    )
+                    if self.report_test_breakdown:
+                        for nodeid in sorted(stats.keys()):
+                            tr._tw.line(f"    {nodeid}")
+                tr._tw.line("")
+
+            runtime_stats = self.get_runtime_stats()
             tr._tw.line(
-                f"Tests   : min {statistics.mininum_tests: >8}, "
-                f"max {statistics.maximum_tests: >8}, "
-                f"average {statistics.average_tests:.1f}"
+                f"Tests   : min {runtime_stats.mininum_tests: >8}, "
+                f"max {runtime_stats.maximum_tests: >8}, "
+                f"average {runtime_stats.average_tests:.1f}"
             )
             tr._tw.line(
-                f"Runtime : min {statistics.mininum_runtime:7.2f}s, "
-                f"max {statistics.maximum_runtime:7.2f}s, "
-                f"average {statistics.average_runtime:.2f}s"
+                f"Runtime : min {runtime_stats.mininum_runtime:7.2f}s, "
+                f"max {runtime_stats.maximum_runtime:7.2f}s, "
+                f"average {runtime_stats.average_runtime:.2f}s"
             )
 
     def pytest_testnodedown(self, node, error):
@@ -88,7 +100,7 @@ def pytest_testnodedown(self, node, error):
             hasattr(node, "workeroutput")
             and (node_worker_stats := node.workeroutput.get(SHARED_WORKER_INFO)) is not None
         ):
-            self.worker_test_times.update(dict(node_worker_stats))
+            self.worker_stats.update(dict(node_worker_stats))
 
     @pytest.hookimpl(hookwrapper=True, trylast=True)
     def pytest_sessionfinish(self, session, exitstatus):
@@ -98,4 +110,4 @@ def pytest_sessionfinish(self, session, exitstatus):
         """
         yield
         if hasattr(self.config, "workeroutput"):
-            self.config.workeroutput[SHARED_WORKER_INFO] = self.worker_test_times
+            self.config.workeroutput[SHARED_WORKER_INFO] = self.worker_stats

tests/conftest.py

@@ -1 +1,42 @@
+import pytest
+
 pytest_plugins = ["pytester"]
+
+
+@pytest.fixture
+def sample_testfile(pytester: pytest.Pytester):
+    code = """
+import pytest
+
+def test_foo():
+    pass
+
+@pytest.mark.parametrize("fix1", (1, 2, 3))
+def test_bar(fix1):
+    pass
+    """
+    pytester.makepyfile(test_plugin=code)
+
+
+expected_header_lines = [
+    "*Worker statistics*",
+]
+
+expected_statistics_lines = [
+    "Tests   : min        2, max        2, average 2.0",
+    "Runtime : min    0.00s, max    0.00s, average 0.00s",
+]
+
+expected_runtime_lines = [
+    "worker gw0  :    2 tests       0.00s runtime",
+    "worker gw1  :    2 tests       0.00s runtime",
+]
+
+expected_breakdown_lines = [
+    "worker gw0  :    2 tests       0.00s runtime",
+    "    test_plugin.py::test_bar[1]",
+    "    test_plugin.py::test_foo",
+    "worker gw1  :    2 tests       0.00s runtime",
+    "    test_plugin.py::test_bar[2]",
+    "    test_plugin.py::test_bar[3]",
+]