[tests] Fix Windows test summary always reporting 'All tests passed' (#24772)

rolfbjarne · Copilot · web-flow · commit 80cbdf04f56f · 2026-02-27T11:17:28.000Z
Fix a copy-paste bug in create-windows-html-report.cs where both the
success and failure branches wrote '# 🎉 All N tests passed 🎉'.
The failure branch now generates a proper summary with &lt;details&gt;/## 
tags listing failed tests, matching the format TestResults.psm1 expects.

Also fix TestResults.psm1 to emit &lt;details&gt; before ##  (correct
HTML nesting order), and avoid including misleading success-format file
content in the failure section when the result file has no failure details.

---------

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/scripts/create-windows-html-report/create-windows-html-report.cs b/scripts/create-windows-html-report/create-windows-html-report.cs
@@ -85,6 +85,10 @@ public static int Main (string [] args)
 
 		var indexContents = new StringBuilder ();
 		var summaryContents = new StringBuilder ();
+		var allFailedTests = new List<(string TrxName, TrxParser.TrxTestResult Test)> ();
+		var failedTrxNames = new List<string> ();
+		var passedTrxCount = 0;
+		var failedTrxCount = 0;
 
 		indexContents.AppendLine ($"<!DOCTYPE html>");
 		indexContents.AppendLine ($"<html>");
@@ -123,9 +127,12 @@ public static int Main (string [] args)
 			var name = trx.Name;
 			var path = trx.TestResults;
 			var messageLines = new List<string> ();
+			var trxSucceeded = true;
 
-			if (TrxParser.TryParseTrxFile (path, out var failedTests, out var outcome, out allTestsSucceeded, out var ex)) {
+			if (TrxParser.TryParseTrxFile (path, out var failedTests, out var outcome, out trxSucceeded, out var ex)) {
 				if (failedTests?.Any () == true) {
+					foreach (var ft in failedTests)
+						allFailedTests.Add ((name, ft));
 					messageLines.Add ("        <ul>");
 					foreach (var ft in failedTests) {
 						var testName = ft.Name;
@@ -152,7 +159,15 @@ public static int Main (string [] args)
 				outcome = "Failed to parse test results";
 				if (ex is not null)
 					messageLines.Add ($"<div>{FormatHtml (ex.ToString ())}</div>");
+				trxSucceeded = false;
+			}
+
+			if (!trxSucceeded) {
 				allTestsSucceeded = false;
+				failedTrxNames.Add (name);
+				failedTrxCount++;
+			} else {
+				passedTrxCount++;
 			}
 
 			try {
@@ -194,7 +209,23 @@ public static int Main (string [] args)
 		if (allTestsSucceeded) {
 			summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
 		} else {
-			summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
+			summaryContents.AppendLine ("# Test results");
+			summaryContents.AppendLine ("<details>");
+			summaryContents.AppendLine ($"<summary>{failedTrxCount} tests failed, {passedTrxCount} tests passed.</summary>");
+			summaryContents.AppendLine ();
+			summaryContents.AppendLine ("## Failed tests");
+			summaryContents.AppendLine ();
+			if (allFailedTests.Any ()) {
+				foreach (var (trxName, test) in allFailedTests) {
+					var msg = string.IsNullOrEmpty (test.Message) ? "" : $": {test.Message.Split ('\n') [0]}";
+					summaryContents.AppendLine ($" * {trxName}/{test.Name}: {test.Outcome}{msg}");
+				}
+			} else {
+				foreach (var trxName in failedTrxNames) {
+					summaryContents.AppendLine ($" * {trxName}: Failed");
+				}
+			}
+			summaryContents.AppendLine ("</details>");
 		}
 
 		Directory.CreateDirectory (outputDirectory);
diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1
@@ -908,8 +908,8 @@ Describe "TestResults tests" {
 
 ### :x: dotnettests tests (MacCatalyst)
 
-<summary>5 tests failed, 6 tests passed.</summary>
 <details>
+<summary>5 tests failed, 6 tests passed.</summary>
 
 </details>
 
@@ -1002,9 +1002,9 @@ Describe "TestResults tests" {
 
 ### :x: dotnettests tests (MacCatalyst)
 
-<summary>1 tests failed, 0 tests passed.</summary>
 <details>
-# :tada: All 5 tests passed :tada:
+<summary>1 tests failed, 0 tests passed.</summary>
+Test results reported success, but the tests job failed.
 </details>
 
 [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip)
diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1
@@ -447,17 +447,21 @@ class ParallelTestsResults {
                         $resultLines = @("Test has no summary file.")
                     }
 
-                    if ($addSummary) {
-                        $stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
-                    }
                     if ($addDetails) {
                         $stringBuilder.AppendLine("<details>")
                     }
-                    if ($startLine -eq -1) {
-                        $startLine = 0
+                    if ($addSummary) {
+                        $stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
                     }
-                    for ($i = $startLine; $i -lt $resultLines.Length; $i++) {
-                        $stringBuilder.AppendLine($resultLines[$i])
+                    if ($startLine -eq -1) {
+                        # No <details>, <summary>, or ## Failed tests found in the file.
+                        # The file likely has the success format (e.g. "# :tada: All N tests passed :tada:"),
+                        # which would be misleading in a failure section. Show a job failure message instead.
+                        $stringBuilder.AppendLine("Test results reported success, but the tests job failed.")
+                    } else {
+                        for ($i = $startLine; $i -lt $resultLines.Length; $i++) {
+                            $stringBuilder.AppendLine($resultLines[$i])
+                        }
                     }
                     if ($addDetails) {
                         $stringBuilder.AppendLine("</details>")
