Skip to content

[tests] Fix Windows test summary always reporting 'All tests passed' #24772

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rolfbjarne merged 3 commits into from
Feb 27, 2026
Merged

[tests] Fix Windows test summary always reporting 'All tests passed' #24772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5f0ee02
[tests] Fix Windows test summary always reporting 'All tests passed'
rolfbjarne Feb 24, 2026
9f585a6
Fix TrxTestResult type reference to TrxParser.TrxTestResult
rolfbjarne Feb 24, 2026
303ae0d
Merge remote-tracking branch 'origin/main' into dev/rolf/windows-not-…
rolfbjarne Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions scripts/create-windows-html-report/create-windows-html-report.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ public static int Main (string [] args)

var indexContents = new StringBuilder ();
var summaryContents = new StringBuilder ();
var allFailedTests = new List<(string TrxName, TrxParser.TrxTestResult Test)> ();
var failedTrxNames = new List<string> ();
var passedTrxCount = 0;
var failedTrxCount = 0;

indexContents.AppendLine ($"<!DOCTYPE html>");
indexContents.AppendLine ($"<html>");
Expand Down Expand Up @@ -123,9 +127,12 @@ public static int Main (string [] args)
var name = trx.Name;
var path = trx.TestResults;
var messageLines = new List<string> ();
var trxSucceeded = true;

if (TrxParser.TryParseTrxFile (path, out var failedTests, out var outcome, out allTestsSucceeded, out var ex)) {
if (TrxParser.TryParseTrxFile (path, out var failedTests, out var outcome, out trxSucceeded, out var ex)) {
if (failedTests?.Any () == true) {
foreach (var ft in failedTests)
allFailedTests.Add ((name, ft));
messageLines.Add (" <ul>");
foreach (var ft in failedTests) {
var testName = ft.Name;
Expand All @@ -152,7 +159,15 @@ public static int Main (string [] args)
outcome = "Failed to parse test results";
if (ex is not null)
messageLines.Add ($"<div>{FormatHtml (ex.ToString ())}</div>");
trxSucceeded = false;
}

if (!trxSucceeded) {
allTestsSucceeded = false;
failedTrxNames.Add (name);
failedTrxCount++;
} else {
passedTrxCount++;
}

try {
Expand Down Expand Up @@ -194,7 +209,23 @@ public static int Main (string [] args)
if (allTestsSucceeded) {
summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
} else {
summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
summaryContents.AppendLine ("# Test results");
summaryContents.AppendLine ("<details>");
summaryContents.AppendLine ($"<summary>{failedTrxCount} tests failed, {passedTrxCount} tests passed.</summary>");
summaryContents.AppendLine ();
summaryContents.AppendLine ("## Failed tests");
summaryContents.AppendLine ();
if (allFailedTests.Any ()) {
foreach (var (trxName, test) in allFailedTests) {
var msg = string.IsNullOrEmpty (test.Message) ? "" : $": {test.Message.Split ('\n') [0]}";
summaryContents.AppendLine ($" * {trxName}/{test.Name}: {test.Outcome}{msg}");
}
} else {
foreach (var trxName in failedTrxNames) {
summaryContents.AppendLine ($" * {trxName}: Failed");
}
}
summaryContents.AppendLine ("</details>");
}

Directory.CreateDirectory (outputDirectory);
Expand Down
6 changes: 3 additions & 3 deletions tools/devops/automation/scripts/TestResults.Tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -908,8 +908,8 @@ Describe "TestResults tests" {
### :x: dotnettests tests (MacCatalyst)
<summary>5 tests failed, 6 tests passed.</summary>
<details>
<summary>5 tests failed, 6 tests passed.</summary>
</details>
Expand Down Expand Up @@ -1002,9 +1002,9 @@ Describe "TestResults tests" {
### :x: dotnettests tests (MacCatalyst)
<summary>1 tests failed, 0 tests passed.</summary>
<details>
# :tada: All 5 tests passed :tada:
<summary>1 tests failed, 0 tests passed.</summary>
Test results reported success, but the tests job failed.
</details>
[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip)
Expand Down
18 changes: 11 additions & 7 deletions tools/devops/automation/scripts/TestResults.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,17 +447,21 @@ class ParallelTestsResults {
$resultLines = @("Test has no summary file.")
}

if ($addSummary) {
$stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
}
if ($addDetails) {
$stringBuilder.AppendLine("<details>")
}
if ($startLine -eq -1) {
$startLine = 0
if ($addSummary) {
$stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
}
for ($i = $startLine; $i -lt $resultLines.Length; $i++) {
$stringBuilder.AppendLine($resultLines[$i])
if ($startLine -eq -1) {
# No <details>, <summary>, or ## Failed tests found in the file.
# The file likely has the success format (e.g. "# :tada: All N tests passed :tada:"),
# which would be misleading in a failure section. Show a job failure message instead.
$stringBuilder.AppendLine("Test results reported success, but the tests job failed.")
} else {
for ($i = $startLine; $i -lt $resultLines.Length; $i++) {
$stringBuilder.AppendLine($resultLines[$i])
}
}
if ($addDetails) {
$stringBuilder.AppendLine("</details>")
Expand Down
Loading