fix(ci-insights): Use pytest-timeout as a deadline (#286)

The goal of this change is to use the timeout as a deadline when
rerunning tests instead of using it to reduce the number of reruns.
Thus, if a rerun take longer to execute, the deadline will work as
expected. We now have a unified behavior for the global and the
pytest-timeout deadlines.

References: MRGFY-6172

Author: remyduthu

pytest_mergify/__init__.py

@@ -224,16 +224,20 @@ def pytest_runtest_protocol(
 
             timeout_seconds = pytest_timeout._get_item_settings(item).timeout
 
+            self.mergify_ci.flaky_detector.set_test_deadline(
+                test=item.nodeid,
+                timeout=datetime.timedelta(seconds=timeout_seconds)
+                if timeout_seconds
+                else None,
+            )
+
             rerun_count = 0
             for _ in range(
-                self.mergify_ci.flaky_detector.get_rerun_count_for_test(
-                    test=item.nodeid,
-                    timeout=datetime.timedelta(seconds=timeout_seconds)
-                    if timeout_seconds
-                    else None,
-                )
+                self.mergify_ci.flaky_detector.get_rerun_count_for_test(item.nodeid)
             ):
-                if self.mergify_ci.flaky_detector.is_deadline_exceeded():
+                if self.mergify_ci.flaky_detector.is_test_deadline_exceeded(
+                    item.nodeid
+                ):
                     break
 
                 for report in self._reruntestprotocol(item, nextitem):
@@ -313,10 +317,9 @@ def pytest_runtest_teardown(
 
         # The goal here is to keep only function-scoped finalizers during
         # reruns and restore higher-scoped finalizers only on the last one.
-        if (
-            self.mergify_ci.flaky_detector.is_deadline_exceeded()
-            or self.mergify_ci.flaky_detector.is_last_rerun_for_test(item.nodeid)
-        ):
+        if self.mergify_ci.flaky_detector.is_test_deadline_exceeded(
+            item.nodeid
+        ) or self.mergify_ci.flaky_detector.is_last_rerun_for_test(item.nodeid):
             self.mergify_ci.flaky_detector.restore_item_finalizers(item)
         else:
             self.mergify_ci.flaky_detector.suspend_item_finalizers(item)

pytest_mergify/flaky_detection.py

@@ -69,6 +69,8 @@ def initial_duration(self) -> datetime.timedelta:
     scheduled_rerun_count: int = dataclasses.field(default=0)
     "Represents the number of reruns that have been scheduled for this test depending on the budget."
 
+    deadline: typing.Optional[datetime.datetime] = dataclasses.field(default=None)
+
     prevented_timeout: bool = dataclasses.field(default=False)
 
     total_duration: datetime.timedelta = dataclasses.field(
@@ -199,9 +201,7 @@ def filter_context_tests_with_session(self, session: _pytest.main.Session) -> No
     def is_test_tracked(self, test: str) -> bool:
         return test in self._test_metrics
 
-    def get_rerun_count_for_test(
-        self, test: str, timeout: typing.Optional[datetime.timedelta] = None
-    ) -> int:
+    def get_rerun_count_for_test(self, test: str) -> int:
         metrics = self._test_metrics.get(test)
         if not metrics:
             return 0
@@ -216,20 +216,7 @@ def get_rerun_count_for_test(
         metrics.is_processed = True
         metrics.scheduled_rerun_count = result
 
-        if not timeout:
-            return result
-
-        # NOTE(remyduthu): Leave a margin of 10 %. Better safe than sorry. We
-        # don't want to crash the CI.
-        safe_timeout = timeout * 0.9
-        if metrics.expected_duration() <= safe_timeout:
-            return result
-
-        metrics.prevented_timeout = True
-
-        # NOTE(remyduthu): Use the timeout to get the number of reruns. The goal
-        # is to still have some reruns.
-        return self._normalize_rerun_count(int(safe_timeout / metrics.initial_duration))
+        return result
 
     def _normalize_rerun_count(self, count: int) -> int:
         result = min(count, self._context.max_test_execution_count)
@@ -239,11 +226,12 @@ def _normalize_rerun_count(self, count: int) -> int:
 
         return result
 
-    def is_deadline_exceeded(self) -> bool:
-        return (
-            self._deadline is not None
-            and datetime.datetime.now(datetime.timezone.utc) >= self._deadline
-        )
+    def is_test_deadline_exceeded(self, test: str) -> bool:
+        metrics = self._test_metrics.get(test)
+        if not metrics or not metrics.deadline:
+            return False
+
+        return datetime.datetime.now(datetime.timezone.utc) >= metrics.deadline
 
     def make_report(self) -> str:
         result = "🐛 Flaky detection"
@@ -335,6 +323,26 @@ def set_deadline(self) -> None:
             + self._get_budget_duration()
         )
 
+    def set_test_deadline(
+        self, test: str, timeout: typing.Optional[datetime.timedelta] = None
+    ) -> None:
+        metrics = self._test_metrics.get(test)
+        if not metrics:
+            return
+
+        metrics.deadline = self._deadline
+
+        if not timeout:
+            return
+
+        # Leave a margin of 10 %. Better safe than sorry. We don't want to crash
+        # the CI.
+        safe_timeout = timeout * 0.9
+        timeout_deadline = datetime.datetime.now(datetime.timezone.utc) + safe_timeout
+        if not metrics.deadline or timeout_deadline < metrics.deadline:
+            metrics.deadline = timeout_deadline
+            metrics.prevented_timeout = True
+
     def is_last_rerun_for_test(self, test: str) -> bool:
         "Returns true if the given test exists and this is its last rerun."
 

tests/test_ci_insights.py

@@ -558,7 +558,9 @@ def pytest_runtest_call(self, item: _pytest.nodes.Item) -> None:
             # Simulate a slow execution that reaches the deadline.
             if not self.deadline_patched and self.execution_count == 10:
                 # Set the deadline in the past to stop immediately.
-                plugin.mergify_ci.flaky_detector._deadline = datetime.datetime.now(
+                plugin.mergify_ci.flaky_detector._test_metrics[
+                    "test_flaky_detection_budget_deadline_stops_reruns.py::test_new"
+                ].deadline = datetime.datetime.now(
                     datetime.timezone.utc
                 ) - datetime.timedelta(hours=1)
 

tests/test_flaky_detection.py

@@ -54,6 +54,26 @@ def _make_flaky_detection_context(
     )
 
 
+@freezegun.freeze_time(_NOW)
+def test_flaky_detector_set_test_deadline() -> None:
+    detector = InitializedFlakyDetector()
+    detector._test_metrics["foo"] = flaky_detection._TestMetrics()
+
+    # Use global deadline by default.
+    detector._deadline = _NOW + datetime.timedelta(seconds=10)
+    detector.set_test_deadline("foo", timeout=None)
+    assert str(detector._test_metrics["foo"].deadline) == "2025-01-01 00:00:10+00:00"
+
+    # Use minimum between global deadline and timeout, if provided.
+    detector.set_test_deadline("foo", timeout=datetime.timedelta(seconds=15))
+    assert str(detector._test_metrics["foo"].deadline) == "2025-01-01 00:00:10+00:00"
+    detector.set_test_deadline("foo", timeout=datetime.timedelta(seconds=5))
+    assert (
+        str(detector._test_metrics["foo"].deadline)
+        == "2025-01-01 00:00:04.500000+00:00"  # 10 % margin applied.
+    )
+
+
 @freezegun.freeze_time(_NOW)
 def test_flaky_detector_get_duration_before_deadline() -> None:
     detector = InitializedFlakyDetector()
@@ -181,29 +201,3 @@ def test_flaky_detector_get_rerun_count_for_test_with_fast_test() -> None:
     detector.set_deadline()
 
     assert detector.get_rerun_count_for_test("foo") == 1000
-
-
-@freezegun.freeze_time(_NOW)
-def test_flaky_detector_get_rerun_count_for_test_with_timeout() -> None:
-    detector = InitializedFlakyDetector()
-    detector._context = _make_flaky_detection_context(
-        min_test_execution_count=5,
-        min_budget_duration_ms=4000,
-        max_test_execution_count=1000,
-    )
-    detector._test_metrics = {
-        "foo": flaky_detection._TestMetrics(
-            initial_call_duration=datetime.timedelta(milliseconds=4),
-        ),
-    }
-    detector.set_deadline()
-
-    # 4 ms * 1000 = 4000 ms budget, but with a timeout of 500 ms and a 10%
-    # safety margin (500 * 0.9 = 450 ms), we can only rerun 112 times (450 ms /
-    # 4 ms).
-    assert (
-        detector.get_rerun_count_for_test(
-            "foo", timeout=datetime.timedelta(milliseconds=500)
-        )
-        == 112
-    )