Merge branch 'master' into issue/1059

Author: paulirwin

.build/azure-templates/publish-test-results.yml

@@ -87,6 +87,13 @@ steps:
                             Write-Host "##vso[task.setvariable variable=CrashedRuns;]$crashedRuns"
                             $crashed = $true
                         }
+                        if ($innerXml -and ($innerXml.Contains('[ERROR]'))) {
+                            Write-Host "##vso[task.setvariable variable=StdOutFailure;]true"
+                            # Report all of the test projects that had stdout failures
+                            $stdOutFailureRuns = "$env:STDOUTFAILURERUNS,$testProjectName".TrimStart(',')
+                            Write-Host "##vso[task.setvariable variable=StdOutFailureRuns;]$stdOutFailureRuns"
+                            $crashed = $true
+                        }
                     }
                 }
                 if ($reader.NodeType -eq [System.Xml.XmlNodeType]::EndElement -and $reader.Name -eq 'RunInfos') {
@@ -102,6 +109,34 @@ steps:
     Write-Host "##vso[task.setvariable variable=TestResultsFileExists;]$testResultsFileExists"
   displayName: 'Parse Test Results File'
 
+- pwsh: |
+    $testProjectName = "${{ parameters.testProjectName }}"
+    $testStdErrFileName = "$(Build.ArtifactStagingDirectory)/${{ parameters.testResultsArtifactName }}/${{ parameters.osName }}/${{ parameters.framework }}/${{ parameters.vsTestPlatform }}/$testProjectName/dotnet-test-error.log"
+    $testStdErrFileExists = Test-Path $testStdErrFileName
+    if ($testStdErrFileExists) {
+        $fileLength = (Get-Item $testStdErrFileName).Length
+        if ($fileLength -gt 0) {
+            $stream = [System.IO.StreamReader]::new($testStdErrFileName)
+            try {
+                while (-not $stream.EndOfStream) {
+                    $line = $stream.ReadLine()
+                    if ($line -match "Test Run Failed" -or $line -match "\[ERROR\]") {
+                        Write-Host "##vso[task.setvariable variable=StdErrFailure;]true"
+                        # Report all of the test projects that had stderr failures
+                        $stdErrFailureRuns = "$env:STDERRFAILURERUNS,$testProjectName".TrimStart(',')
+                        Write-Host "##vso[task.setvariable variable=StdErrFailureRuns;]$stdErrFailureRuns"
+                        break # No need to continue reading after detecting a failure
+                    }
+                }
+            } finally {
+                $stream.Dispose()
+            }
+        }
+    } else {
+        Write-Host "WARNING: File not found: $testStdErrFileName"
+    }
+  displayName: 'Parse StdErr File'
+
 - task: PublishTestResults@2
   displayName: 'Publish Test Results ${{ parameters.testProjectName }},${{ parameters.framework }},${{ parameters.vsTestPlatform }}'
   inputs:

.build/azure-templates/run-tests-on-os.yml

@@ -19,7 +19,7 @@
 # runs the tests for each project on a background job in parallel,
 # then uploads the results to Azure DevOps pipelines
 
-# NOTE: Depends on environment variables $(IsNightly) and $(IsWeekly)
+# NOTE: Depends on environment variables $(IsNightly), $(IsWeekly), and $(DisplayFullName)
 
 parameters:
   osName: 'Windows' # The name of the operating system for display purposes.
@@ -172,6 +172,7 @@ steps:
     $tempDirectory = "$(Agent.TempDirectory)"
     $isNightly = if ($env:ISNIGHTLY -eq 'true') { 'true' } else { 'false' }
     $isWeekly = if ($env:ISWEEKLY -eq 'true') { 'true' } else { 'false' }
+    $displayFullName = if ($env:DISPLAYFULLNAME -eq 'false') { 'false' } else { 'true' }
 
     function IsSupportedFramework([string]$framework) {
         if ($IsWindows -eq $null) {
@@ -226,6 +227,10 @@ steps:
 
             $testExpression = "$testExpression -- RunConfiguration.TargetPlatform=$testPlatform"
 
+            if ($displayFullName -eq 'true') {
+                $testExpression = "$testExpression NUnit.DisplayName=FullName"
+            }
+
             Write-Host "Testing '$($testBinary.FullName)' on framework '$framework' and outputting test results to '$testResultDirectory/$testResultsFileName'..."
             Write-Host $testExpression -ForegroundColor Magenta
             if ($maximumParalellJobs -le 1) {
@@ -305,6 +310,16 @@ steps:
         Write-Host "##vso[task.logissue type=error;]Test run failed due to too many failed tests. Maximum failures allowed: $maximumAllowedFailures, total failures: $($env:TOTALFAILURES)."
         $failed = $true
     }
+    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDERRFAILURE -eq 'true') {
+        $runsExpanded = "$($env:STDERRFAILURERUNS)" -replace ',',"`r`n"
+        Write-Host "##vso[task.logissue type=error;]StdErr file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
+        $failed = $true
+    }
+    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDOUTFAILURE -eq 'true') {
+        $runsExpanded = "$($env:STDOUTFAILURERUNS)" -replace ',',"`r`n"
+        Write-Host "##vso[task.logissue type=error;]StdOut file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
+        $failed = $true
+    }
     if ($failed) {
         Write-Host "##vso[task.complete result=Failed;]"
     }

.build/runbuild.ps1

@@ -328,6 +328,12 @@ task Test -depends CheckSDK, UpdateLocalSDKVersion, Restore -description "This t
                 $testExpression = "$testExpression --TestCaseFilter:""$where"""
             }
 
+            # Anything after here is test run settings, following the "--" separator
+            $testExpression = "$testExpression --"
+
+            # Specify NUnit.DisplayName to get the full test class name in the output
+            $testExpression = "$testExpression NUnit.DisplayName=FullName"
+
             Write-Host $testExpression -ForegroundColor Magenta
 
             $scriptBlock = {

.github/workflows/Generate-TestWorkflows.ps1

@@ -232,6 +232,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'$directoryBuildPaths
     # Dependencies$dependencyPaths
     - '!**/*.md'
@@ -322,7 +323,7 @@ jobs:
 
     $fileText += "
       - run: dotnet build `${{env.project_path}} --configuration `${{matrix.configuration}} --framework `${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test `${{env.project_path}} --configuration `${{matrix.configuration}} --framework `${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:`"console;verbosity=normal`" --logger:`"trx;LogFileName=`${{env.trx_file_name}}`" --logger:`"liquid.md;LogFileName=`${{env.md_file_name}};Title=`${{env.title}};`" --results-directory:`"`${{github.workspace}}/`${{env.test_results_artifact_name}}/`${{env.project_name}}`" -- RunConfiguration.TargetPlatform=`${{matrix.platform}} TestRunParameters.Parameter\(name=\`"tests:slow\`",\ value=\`"\`${{env.run_slow_tests}}\`"\)
+      - run: dotnet test `${{env.project_path}} --configuration `${{matrix.configuration}} --framework `${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:`"console;verbosity=normal`" --logger:`"trx;LogFileName=`${{env.trx_file_name}}`" --logger:`"liquid.md;LogFileName=`${{env.md_file_name}};Title=`${{env.title}};`" --results-directory:`"`${{github.workspace}}/`${{env.test_results_artifact_name}}/`${{env.project_name}}`" -- RunConfiguration.TargetPlatform=`${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\`"tests:slow\`",\ value=\`"\`${{env.run_slow_tests}}\`"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-AllProjects.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.AllProjects/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -132,7 +133,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-Analysis-Common.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.Analysis.Common/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -116,7 +117,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-Analysis-Kuromoji.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.Analysis.Kuromoji/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -113,7 +114,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-Analysis-Morfologik.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.Analysis.Morfologik/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -113,7 +114,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-Analysis-OpenNLP.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.Analysis.OpenNLP/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -116,7 +117,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact

.github/workflows/Lucene-Net-Tests-Analysis-Phonetic.yml

@@ -28,6 +28,7 @@ on:
     - '.build/dependencies.props'
     - '.build/TestReferences.Common.*'
     - 'TestTargetFrameworks.*'
+    - '.github/**/*.yml'
     - '*.sln'
     - 'src/Lucene.Net.Tests.Analysis.Phonetic/Directory.Build.*'
     - 'src/Directory.Build.*'
@@ -110,7 +111,7 @@ jobs:
           echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath  $env:GITHUB_ENV -Encoding utf8 -Append
         shell: pwsh
       - run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
-      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
+      - run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
         shell: bash
       # upload reports as build artifacts
       - name: Upload a Build Artifact