Parallelize tests

eng/Versions.props

@@ -184,7 +184,7 @@
     <MicrosoftNETCoreAppRefVersion>3.1.0</MicrosoftNETCoreAppRefVersion>
     <MicrosoftNETCoreILDAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILDAsmVersion>
     <MicrosoftNETCoreILAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILAsmVersion>
-    <MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
+    <MicrosoftNETTestSdkVersion>17.11.1</MicrosoftNETTestSdkVersion>
     <NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
     <RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
     <StreamJsonRpcVersion>2.18.48</StreamJsonRpcVersion>

eng/build.sh

@@ -214,7 +214,8 @@ function Test() {
   projectname=$(basename -- "$testproject")
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
-  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args+=" -- xUnit.MaxParallelThreads=4"
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

src/Compiler/Utilities/illib.fs

@@ -137,7 +137,7 @@ module internal PervasiveAutoOpens =
     type Async with
 
         static member RunImmediate(computation: Async<'T>, ?cancellationToken) =
-            let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken
+            let cancellationToken = defaultArg cancellationToken CancellationToken.None
 
             let ts = TaskCompletionSource<'T>()
 

tests/EndToEndBuildTests/BasicProvider/BasicProvider.Tests/BasicProvider.Tests.fsproj

@@ -21,7 +21,7 @@
         <PackagePath>content\myfiles\</PackagePath>
     </Content>
     <PackageReference Include="BasicProvider" Version="1.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
   </ItemGroup>
 
   <Target Name="RemovePackagesFromCache" BeforeTargets="Restore">

tests/EndToEndBuildTests/ComboProvider/ComboProvider.Tests/ComboProvider.Tests.fsproj

@@ -18,7 +18,8 @@
     </None>
 
     <PackageReference Include="ComboProvider" Version="1.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+
   </ItemGroup>
 
   <Target Name="RemovePackagesFromCache" BeforeTargets="Restore">

tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

@@ -4,17 +4,26 @@
 
   <PropertyGroup>
     <TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
     <OutputType>Library</OutputType>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <UnitTestType>xunit</UnitTestType>
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="WriteCodeFragmentTests.fs" />
     <Compile Include="MapSourceRootsTests.fs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content> 
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />

tests/FSharp.Build.UnitTests/xunit.runner.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeAssembly": true
+}

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs

@@ -7,6 +7,7 @@ open FSharp.Test
 open FSharp.Test.Compiler
 open System
 
+[<Collection(nameof DoNotRunInParallel)>]
 module utf8output =
 
     [<Fact>]

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs

@@ -8,6 +8,8 @@ open FSharp.Test.Compiler
 open System
 open System.IO
 
+// reportTime uses global state.
+[<Collection(nameof DoNotRunInParallel)>]
 module times =
 
     // This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs

@@ -14,5 +14,5 @@ let success =
     asm.Version.Major = 1 &&
     asm.Version.Minor = 2 &&
     asm.Version.Build = 3 &&
-    (abs (asm.Version.Revision - (int defaultRevision))) < 10  // default value is seconds in the current day / 2.  Check if within 10 sec of that.
+    (abs (asm.Version.Revision - (int defaultRevision))) < 60  // default value is seconds in the current day / 2.  Check if within 60 sec of that.
 if success then () else failwith "Failed: 1"

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs

@@ -6,6 +6,7 @@ open Xunit
 open FSharp.Test
 open FSharp.Test.Compiler
 
+[<Collection(nameof DoNotRunInParallel)>]
 module Events =
 
     let verifyCompile compilation =

tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs

@@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
     | CompilationResult.Success ({OutputPath = Some dllFile} as s) -> 
        let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
        File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
-       let _exitCode, _stdout, stderr, _exn =  CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
+       let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
 
-       Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
+       Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")
 
     | _ -> failwith (sprintf "%A" compilationResult)
 

tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj

@@ -28,6 +28,9 @@
 </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="CompilerDirectives\Line.fs" />
     <Compile Include="CompilerDirectives\Ifdef.fs" />
     <Compile Include="CompilerDirectives\NonStringArgs.fs" />

tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

@@ -13,6 +13,7 @@ open FSharp.Compiler.Diagnostics
 
 open Xunit
 
+open FSharp.Test
 open FSharp.Test.ProjectGeneration
 open FSharp.Test.ProjectGeneration.Helpers
 open System.IO
@@ -1064,67 +1065,71 @@ type private LoadClosureTestShim(currentFileSystem: IFileSystem) =
                 ?shouldShadowCopy = shouldShadowCopy
             )
 
-[<Theory>]
-[<InlineData(false)>]
-[<InlineData(true)>]
-let ``The script load closure should always be evaluated`` useTransparentCompiler =
-    async {
-        // The LoadScriptClosure uses the file system shim so we need to reset that.
-        let currentFileSystem = FileSystemAutoOpens.FileSystem
-        let assumeDotNetFramework =
-            // The old BackgroundCompiler uses assumeDotNetFramework = true
-            // This is not always correctly loading when this test runs on non-Windows.
-            if System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") then
-                None
-            else
-                Some false 
+// Because it is mutating FileSystem!
+[<Collection(nameof DoNotRunInParallel)>]
+module TestsMutatingFileSystem =
+
+    [<Theory>]
+    [<InlineData(false)>]
+    [<InlineData(true)>]
+    let ``The script load closure should always be evaluated`` useTransparentCompiler =
+        async {
+            // The LoadScriptClosure uses the file system shim so we need to reset that.
+            let currentFileSystem = FileSystemAutoOpens.FileSystem
+            let assumeDotNetFramework =
+                // The old BackgroundCompiler uses assumeDotNetFramework = true
+                // This is not always correctly loading when this test runs on non-Windows.
+                if System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") then
+                    None
+                else
+                    Some false 
 
-        try
-            let checker = FSharpChecker.Create(useTransparentCompiler = useTransparentCompiler)
-            let fileSystemShim = LoadClosureTestShim(currentFileSystem)
-            // Override the file system shim for loading b.fsx
-            FileSystem <- fileSystemShim
-
-            let! initialSnapshot, _ =
-                checker.GetProjectSnapshotFromScript(
-                    "a.fsx",
-                    SourceTextNew.ofString fileSystemShim.aFsx,
-                    documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
-                    ?assumeDotNetFramework = assumeDotNetFramework
-                )
-
-            // File b.fsx should also be included in the snapshot.
-            Assert.Equal(2, initialSnapshot.SourceFiles.Length)
-
-            let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", initialSnapshot)
-
-            match snd checkResults with
-            | FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
-            | FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
+            try
+                let checker = FSharpChecker.Create(useTransparentCompiler = useTransparentCompiler)
+                let fileSystemShim = LoadClosureTestShim(currentFileSystem)
+                // Override the file system shim for loading b.fsx
+                FileSystem <- fileSystemShim
+
+                let! initialSnapshot, _ =
+                    checker.GetProjectSnapshotFromScript(
+                        "a.fsx",
+                        SourceTextNew.ofString fileSystemShim.aFsx,
+                        documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
+                        ?assumeDotNetFramework = assumeDotNetFramework
+                    )
+
+                // File b.fsx should also be included in the snapshot.
+                Assert.Equal(2, initialSnapshot.SourceFiles.Length)
+
+                let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", initialSnapshot)
+
+                match snd checkResults with
+                | FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
+                | FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
 
-            // Update b.fsx, it should now load c.fsx
-            fileSystemShim.UpdateB()
-
-            // The constructed key for the load closure will the exactly the same as the first GetProjectSnapshotFromScript call.
-            // However, a none cached version will be computed first in GetProjectSnapshotFromScript and update the cache afterwards.
-            let! secondSnapshot, _ =
-                checker.GetProjectSnapshotFromScript(
-                    "a.fsx",
-                    SourceTextNew.ofString fileSystemShim.aFsx,
-                    documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
-                    ?assumeDotNetFramework = assumeDotNetFramework
-                )
-
-            Assert.Equal(3, secondSnapshot.SourceFiles.Length)
-
-            let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", secondSnapshot)
-
-            match snd checkResults with
-            | FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
-            | FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
-        finally
-            FileSystemAutoOpens.FileSystem <- currentFileSystem
-    }
+                // Update b.fsx, it should now load c.fsx
+                fileSystemShim.UpdateB()
+
+                // The constructed key for the load closure will the exactly the same as the first GetProjectSnapshotFromScript call.
+                // However, a none cached version will be computed first in GetProjectSnapshotFromScript and update the cache afterwards.
+                let! secondSnapshot, _ =
+                    checker.GetProjectSnapshotFromScript(
+                        "a.fsx",
+                        SourceTextNew.ofString fileSystemShim.aFsx,
+                        documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
+                        ?assumeDotNetFramework = assumeDotNetFramework
+                    )
+
+                Assert.Equal(3, secondSnapshot.SourceFiles.Length)
+
+                let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", secondSnapshot)
+
+                match snd checkResults with
+                | FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
+                | FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
+            finally
+                FileSystemAutoOpens.FileSystem <- currentFileSystem
+        }
 
 [<Fact>]
 let ``Parsing without cache and without project snapshot`` () =

tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs

@@ -1,5 +1,7 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
+// Because of shared fsi session.
+[<FSharp.Test.RunInSequence>]
 module Language.BooleanReturningAndReturnTypeDirectedPartialActivePatternTests
 
 open Xunit

tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs

@@ -1,5 +1,7 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
+// Run sequentially because of shared fsiSession.
+[<FSharp.Test.RunInSequence>]
 module Language.SequenceExpression.SequenceExpressionTests
 
 open FSharp.Test

tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs

@@ -31,17 +31,20 @@ module ScriptRunner =
         let cu  = cu |> withDefines defaultDefines
         match cu with 
         | FS fsSource ->
+            use capture = new TestConsole.ExecutionCapture()
             let engine = createEngine (fsSource.Options |> Array.ofList,version)
             let res = evalScriptFromDiskInSharedSession engine cu
             match res with
             | CompilationResult.Failure _ -> res
-            | CompilationResult.Success s ->
-                if engine.GetOutput().Contains "TEST PASSED OK" then
+            | CompilationResult.Success _ ->               
+                if capture.OutText |> TestFramework.outputPassed then
                     res
                 else
-                    failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"       
+                    failwith $"Results looked correct, but 'TEST PASSED OK' was not printed."       
 
-        | _ -> failwith $"Compilation unit other than fsharp is not supported, cannot process %A{cu}"
+        | _ ->
+            printfn $"Cannot process %A{cu}"
+            failwith $"Compilation unit other than fsharp is not supported."
 
 /// This test file was created by porting over (slower) FsharpSuite.Tests
 /// In order to minimize human error, the test definitions have been copy-pasted and this adapter provides implementations of the test functions

tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs

@@ -93,14 +93,17 @@ let ``comprehensions-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehens
 [<Fact>]
 let ``comprehensions-FSI`` () = singleTestBuildAndRun "core/comprehensions" FSI
 
-[<Fact>]
-let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG
+// Cancels default token.
+[<Collection(nameof DoNotRunInParallel)>]
+module Comprehensionshw =
+    [<Fact>]
+    let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG
 
-[<Fact>]
-let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED
+    [<Fact>]
+    let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED
 
-[<Fact>]
-let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI
+    [<Fact>]
+    let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI
 
 [<Fact>]
 let ``genericmeasures-FSC_DEBUG`` () = singleTestBuildAndRun "core/genericmeasures" FSC_DEBUG
@@ -375,18 +378,21 @@ let ``recordResolution-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/recordRe
 [<Fact>]
 let ``recordResolution-FSI`` () = singleTestBuildAndRun "core/recordResolution" FSI
 
-// This test has hardcoded expectations about current synchronization context
-// Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
-[<FactForNETCOREAPP>]
-let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED
+// Cancels default token.
+[<Collection(nameof DoNotRunInParallel)>]
+module CoreControl =
+    // This test has hardcoded expectations about current synchronization context
+    // Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
+    [<FactForNETCOREAPP>]
+    let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED
 
-[<FactForNETCOREAPP>]
-let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI
+    [<FactForNETCOREAPP>]
+    let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI
 
-[<FactForNETCOREAPP>]
-let ``control --tailcalls`` () =
-    let cfg =  "core/control"
-    singleTestBuildAndRunAux cfg  ["--tailcalls"] FSC_OPTIMIZED
+    [<FactForNETCOREAPP>]
+    let ``control --tailcalls`` () =
+        let cfg =  "core/control"
+        singleTestBuildAndRunAux cfg  ["--tailcalls"] FSC_OPTIMIZED
 
 [<Fact>]
 let ``controlChamenos-FSC_OPTIMIZED`` () =
@@ -401,7 +407,7 @@ let ``controlChamenos-FSI`` () =
 [<Fact>]
 let ``controlMailbox-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/controlMailbox" FSC_OPTIMIZED
 
-[<Fact(Skip="Flaky")>]
+[<Fact>]
 let ``controlMailbox-FSI`` () = singleTestBuildAndRun "core/controlMailbox" FSI
 
 [<Fact>]

tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs

@@ -5,7 +5,7 @@ open FSharp.Compiler.GraphChecking
 open FSharp.Compiler.Text
 open FSharp.Compiler.Syntax
 
-let private checker = FSharpChecker.Create()
+open FSharp.Test
 
 let parseSourceCode (name: string, code: string) =
     let sourceText = SourceText.ofString code
@@ -16,7 +16,7 @@ let parseSourceCode (name: string, code: string) =
         }
 
     let result =
-        checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously
+        TestContext.Checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously
 
     result.ParseTree