chore(ci-insights): Log reruns in pytest report (#282)

The goal of this change is to make reruns visible in the final pytest
report. It'll help us debug issues with the flaky detection mechanism.

References: MRGFY-6172

Author: remyduthu

pytest_mergify/__init__.py

@@ -272,10 +272,34 @@ def _reruntestprotocol(
             item=item, nextitem=nextitem, log=False
         )
         for report in reports:
-            self.mergify_ci.flaky_detector.try_fill_metrics_from_report(report)
+            if report.when != "call":
+                item.ihook.pytest_runtest_logreport(report=report)  # Log as usual.
+            else:
+                # Make rerun visible in the logs by temporarily changing
+                # outcome. The goal is to count a potential failure as a rerun
+                # instead of a regular failure.
+                original_outcome = report.outcome
+                report.outcome = "rerun"  # type: ignore[assignment]
+                item.ihook.pytest_runtest_logreport(report=report)
+                report.outcome = original_outcome
 
         return reports
 
+    @pytest.hookimpl
+    def pytest_report_teststatus(
+        self,
+        report: _pytest.reports.TestReport,
+    ) -> typing.Optional[
+        typing.Tuple[
+            str, str, typing.Union[str, typing.Tuple[str, typing.Mapping[str, bool]]]
+        ]
+    ]:
+        # https://github.com/pytest-dev/pytest-rerunfailures/blob/master/src/pytest_rerunfailures.py#L622-L625
+        if report.outcome == "rerun":  # type: ignore[comparison-overlap]
+            return "rerun", "R", ("RERUN", {"yellow": True})  # type: ignore[unreachable]
+
+        return None
+
     @pytest.hookimpl(tryfirst=True)
     def pytest_runtest_teardown(
         self,

pytest_mergify/flaky_detection.py

@@ -166,7 +166,7 @@ def _fetch_context(self) -> _FlakyDetectionContext:
         return result
 
     def try_fill_metrics_from_report(self, report: _pytest.reports.TestReport) -> bool:
-        if report.outcome not in ["failed", "passed"]:
+        if report.outcome not in ["failed", "passed", "rerun"]:
             return False
 
         test = report.nodeid

tests/test_ci_insights.py

@@ -269,10 +269,12 @@ def test_quux():
     # The goal is to make sure the failed rerun of the flaky test does not
     # impact the results to avoid failing the CI of our users.
     assert result.ret == 0
-    result.assert_outcomes(
-        passed=4,  # Initial run of each test.
-        skipped=1,
-    )
+
+    outcomes = result.parseoutcomes()
+    assert len(outcomes) == 3
+    assert outcomes["passed"] == 4  # Initial run of each test.
+    assert outcomes["skipped"] == 1
+    assert outcomes["rerun"] == 3000  # 1000 reruns for each unhealthy test.
 
     assert re.search(
         r"""🐛 Flaky detection