Deprecate passing iterator argvalues to parametrize

Fix #13409.

Author: bluetech

changelog/13409.deprecation.rst

@@ -0,0 +1,8 @@
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects) for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>` and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is now deprecated.
+
+These iterables get exhausted after the first iteration,
+leading to tests getting unexpectedly skipped in cases such as running :func:`pytest.main()` multiple times,
+using class-level parametrize decorators,
+or collecting tests multiple times.
+
+See :ref:`parametrize-iterators` for details and suggestions.

doc/en/deprecations.rst

@@ -15,6 +15,67 @@ Below is a complete list of all pytest features which are considered deprecated.
 :class:`~pytest.PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters <warnings>`.
 
 
+.. _parametrize-iterators:
+
+Non-Collection iterables in ``@pytest.mark.parametrize``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. deprecated:: 9.1
+
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects)
+for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>`
+and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is deprecated.
+
+These iterables get exhausted after the first iteration, leading to tests getting unexpectedly skipped in cases such as:
+
+* Running :func:`pytest.main()` multiple times in the same process
+* Using class-level parametrize decorators where the same mark is applied to multiple test methods
+* Collecting tests multiple times
+
+Example of problematic code:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", data_generator())
+    class Test:
+        def test_1(self, n):
+            pass
+
+        # test_2 will be skipped because data_generator() is exhausted.
+        def test_2(self, n):
+            pass
+
+You can fix it by convert generators and iterators to lists or tuples:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", list(data_generator()))
+    class Test:
+        def test_1(self, n):
+            pass
+
+        def test_2(self, n):
+            pass
+
+Note that :class:`range` objects are ``Collection`` and are not affected by this deprecation.
+
+
 .. _monkeypatch-fixup-namespace-packages:
 
 ``monkeypatch.syspath_prepend`` with legacy namespace packages

src/_pytest/compat.py

@@ -15,6 +15,7 @@
 from typing import Any
 from typing import Final
 from typing import NoReturn
+from typing import TYPE_CHECKING
 
 import py
 
@@ -311,3 +312,17 @@ def running_on_ci() -> bool:
     # Only enable CI mode if one of these env variables is defined and non-empty.
     env_vars = ["CI", "BUILD_NUMBER"]
     return any(os.environ.get(var) for var in env_vars)
+
+
+if sys.version_info >= (3, 13):
+    from warnings import deprecated as deprecated
+else:
+    if TYPE_CHECKING:
+        from typing_extensions import deprecated as deprecated
+    else:
+
+        def deprecated(msg, /, *, category=None, stacklevel=1):
+            def decorator(func):
+                return func
+
+            return decorator

src/_pytest/deprecated.py

@@ -75,6 +75,14 @@
     "See https://docs.pytest.org/en/stable/deprecations.html#monkeypatch-fixup-namespace-packages"
 )
 
+PARAMETRIZE_NON_COLLECTION_ITERABLE = UnformattedWarning(
+    PytestRemovedIn10Warning,
+    "Passing a non-Collection iterable to parametrize is deprecated.\n"
+    "Test: {nodeid}, argvalues type: {type_name}\n"
+    "Please convert to a list or tuple.\n"
+    "See https://docs.pytest.org/en/stable/deprecations.html#parametrize-iterators",
+)
+
 # You want to make some `__init__` or function "private".
 #
 #   def my_private_function(some, args):

src/_pytest/mark/structures.py

@@ -21,11 +21,13 @@
 import warnings
 
 from .._code import getfslineno
+from ..compat import deprecated
 from ..compat import NOTSET
 from ..compat import NotSetType
 from _pytest.config import Config
 from _pytest.deprecated import check_ispytest
 from _pytest.deprecated import MARKED_FIXTURE
+from _pytest.deprecated import PARAMETRIZE_NON_COLLECTION_ITERABLE
 from _pytest.outcomes import fail
 from _pytest.raises import AbstractRaises
 from _pytest.scope import _ScopeName
@@ -193,6 +195,15 @@ def _for_parametrize(
         config: Config,
         nodeid: str,
     ) -> tuple[Sequence[str], list[ParameterSet]]:
+        if not isinstance(argvalues, Collection):
+            warnings.warn(
+                PARAMETRIZE_NON_COLLECTION_ITERABLE.format(
+                    nodeid=nodeid,
+                    type_name=type(argvalues).__name__,
+                ),
+                stacklevel=3,
+            )
+
         argnames, force_tuple = cls._parse_parametrize_args(argnames, argvalues)
         parameters = cls._parse_parametrize_parameters(argvalues, force_tuple)
         del argvalues
@@ -508,7 +519,25 @@ def __call__(
         ) -> MarkDecorator: ...
 
     class _ParametrizeMarkDecorator(MarkDecorator):
-        def __call__(  # type: ignore[override]
+        @overload  # type: ignore[override,no-overload-impl]
+        def __call__(
+            self,
+            argnames: str | Sequence[str],
+            argvalues: Collection[ParameterSet | Sequence[object] | object],
+            *,
+            indirect: bool | Sequence[str] = ...,
+            ids: Iterable[None | str | float | int | bool]
+            | Callable[[Any], object | None]
+            | None = ...,
+            scope: _ScopeName | None = ...,
+        ) -> MarkDecorator: ...
+
+        @overload
+        @deprecated(
+            "Passing a non-Collection iterable to the 'argvalues' parameter of @pytest.mark.parametrize is deprecated. "
+            "Convert argvalues to a list or tuple.",
+        )
+        def __call__(
             self,
             argnames: str | Sequence[str],
             argvalues: Iterable[ParameterSet | Sequence[object] | object],

src/_pytest/python.py

@@ -1247,6 +1247,12 @@ def parametrize(
             N-tuples, where each tuple-element specifies a value for its
             respective argname.
 
+            .. versionchanged:: 9.1
+
+                Passing a non-:class:`~collections.abc.Collection` iterable
+                (such as a generator or iterator) is deprecated. See
+                :ref:`parametrize-iterators` for details.
+
         :param indirect:
             A list of arguments' names (subset of argnames) or a boolean.
             If True the list contains all names from the argnames. Each

testing/python/metafunc.py

@@ -1123,6 +1123,24 @@ def test_3(self, arg, arg2):
         """
         )
 
+    def test_parametrize_iterator_deprecation(self) -> None:
+        """Test that using iterators for argvalues raises a deprecation warning."""
+
+        def func(x: int) -> None:
+            raise NotImplementedError()
+
+        def data_generator() -> Iterator[int]:
+            yield 1
+            yield 2
+
+        metafunc = self.Metafunc(func)
+
+        with pytest.warns(
+            pytest.PytestRemovedIn10Warning,
+            match=r"Passing a non-Collection iterable to parametrize is deprecated",
+        ):
+            metafunc.parametrize("x", data_generator())
+
 
 class TestMetafuncFunctional:
     def test_attributes(self, pytester: Pytester) -> None:
@@ -1682,6 +1700,65 @@ def test_3(self, fixture):
             ]
         )
 
+    def test_parametrize_generator_multiple_runs(self, pytester: Pytester) -> None:
+        """Test that generators in parametrize work with multiple pytest.main() (deprecated)."""
+        testfile = pytester.makepyfile(
+            """
+            import pytest
+
+            def data_generator():
+                yield 1
+                yield 2
+
+            @pytest.mark.parametrize("bar", data_generator())
+            def test_foo(bar):
+                pass
+
+            if __name__ == '__main__':
+                args = ["-q", "--collect-only"]
+                pytest.main(args)  # First run - should work with warning
+                pytest.main(args)  # Second run - should also work with warning
+            """
+        )
+        result = pytester.run(sys.executable, "-Wdefault", testfile)
+        # Should see the deprecation warnings.
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestRemovedIn10Warning: Passing a non-Collection iterable*",
+                "*PytestRemovedIn10Warning: Passing a non-Collection iterable*",
+            ]
+        )
+
+    def test_parametrize_iterator_class_multiple_tests(
+        self, pytester: Pytester
+    ) -> None:
+        """Test that iterators in parametrize on a class get exhausted (deprecated)."""
+        pytester.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("n", iter(range(2)))
+            class Test:
+                def test_1(self, n):
+                    pass
+
+                def test_2(self, n):
+                    pass
+            """
+        )
+        result = pytester.runpytest("-v", "-Wdefault")
+        # Iterator gets exhausted after first test, second test gets no parameters.
+        # This is deprecated.
+        result.assert_outcomes(passed=2, skipped=1)
+        result.stdout.fnmatch_lines(
+            [
+                "*test_parametrize_iterator_class_multiple_tests.py::Test::test_1[[]0] PASSED*",
+                "*test_parametrize_iterator_class_multiple_tests.py::Test::test_1[[]1] PASSED*",
+                "*test_parametrize_iterator_class_multiple_tests.py::Test::test_2[[]NOTSET] SKIPPED*",
+                "*PytestRemovedIn10Warning: Passing a non-Collection iterable*",
+            ]
+        )
+
 
 class TestMetafuncFunctionalAuto:
     """Tests related to automatically find out the correct scope for

testing/typing_checks.py

@@ -58,3 +58,10 @@ def check_raises_is_a_context_manager(val: bool) -> None:
 def check_testreport_attributes(report: TestReport) -> None:
     assert_type(report.when, Literal["setup", "call", "teardown"])
     assert_type(report.location, tuple[str, int | None, str])
+
+
+# Test @pytest.mark.parametrize iterator argvalues deprecation.
+# Will be complain about unused type ignore if doesn't work.
+@pytest.mark.parametrize("x", iter(range(10)))  # type: ignore[deprecated]
+def test_it(x: int) -> None:
+    pass