Fix for continuous testing

...and code coverage and dotMemoryUnit. Disables parallelisation and
async test message reporting, so that the monitoring tools know which
test executing code belongs to, and knows exactly when a tests
starts/finishes.

Author: citizenmatt

resharper/CommonAssemblyInfo.cs

@@ -6,4 +6,4 @@
 [assembly : AssemblyDescription("xUnit.net unit test provider for " + ProductInfo.Product)]
 [assembly : AssemblyCopyright("Copyright (C) Matt Ellis")]
 [assembly : ComVisible(false)]
-[assembly : AssemblyVersion("2.3.1.0")]
+[assembly : AssemblyVersion("2.3.2.0")]

resharper/nuget/xunitcontrib.nuspec

@@ -3,15 +3,16 @@
   <metadata>
     <id>CitizenMatt.Xunit</id>
     <title>xUnit.net Test Support for ReSharper 10</title>
-    <version>2.3.1</version>
+    <version>2.3.2</version>
     <authors>Matt Ellis</authors>
     <owners>Matt Ellis</owners>
     <description>A unit test provider for xUnit.net. Discovers and runs xUnit.net 1.x and 2.x tests. Includes annotations to aid ReSharper inspections and Live Templates to speed up inserting test methods and asserts.</description>
     <summary>A unit test provider for xUnit.net</summary>
     <releaseNotes>
-&#8226; Fixed issue with discovering theories at runtime
+&#8226; Now works properly with code coverage, continuous testing + dotMemoryUnit, by disabling all concurrency under these conditions
 
 From previous releases:
+&#8226; Fixed issue with discovering theories at runtime
 &#8226; Support for ReSharper 10 (#82)
 &#8226; Inherited tests not showing in results (#79)
 &#8226; Theories can now be excluded by category (#89)

resharper/src/provider/UnitTestElements/XunitTestClassElement.cs

@@ -77,15 +77,38 @@ where gc.State.IsValid()
             var knownChildren = new HashSet<string>(knownMethods);
             knownChildren.AddRange(knownTheories);
 
+            var disableAllConcurrency = ShouldDisableAllConcurrency(run);
+
             var projectId = Id.Project.GetPersistentID();
             return new List<UnitTestTask>
                        {
-                           new UnitTestTask(null, new XunitBootstrapTask(projectId)),
+                           new UnitTestTask(null, new XunitBootstrapTask(projectId, disableAllConcurrency)),
                            new UnitTestTask(null, new XunitTestAssemblyTask(projectId, AssemblyLocation)),
                            new UnitTestTask(this, new XunitTestClassTask(projectId, TypeName.FullName, explicitElements.Contains(this), knownChildren))
                        };
         }
 
+        private bool ShouldDisableAllConcurrency(IUnitTestRun run)
+        {
+            // Code coverage (and therefore continuous testing) and dotMemoryUnit cannot handle
+            // tests running concurrently (code coverage and memory usage need to be tied back
+            // to a specific test), so we need to disable concurrency in these environments. For
+            // xunit, this means disabling parallelisation and async reporting of test messages.
+            // This is likely to be set automatically in the test runner process by the ReSharper
+            // test hosts - TaskExecutorConfiguration.DisallowTestConcurrency or some such.
+            // See https://youtrack.jetbrains.com/issue/DCVR-7804
+            switch (run.Launch.HostProvider.ID)
+            {
+                // TODO: It would be nice to use the actual constants, but they're not referenced
+                // case ContinuousTestingHostProvider.ContinuousTestingHostProviderId:
+                case "ContinuousTestingHostProviderId":
+                case "Cover":
+                case "dotMemoryUnit":
+                    return true;
+            }
+            return false;
+        }
+
         public override string Kind
         {
             get { return "xUnit.net Test Class"; }

resharper/src/runner/Executor.cs

@@ -27,6 +27,13 @@ public void RunTests(IList<ITestCase> testCases)
 
             var options = TestFrameworkOptions.ForExecution(environment.TestAssemblyConfiguration);
 
+            // If the test hosts have requested all concurrency to be disabled, make sure we
+            // also make reporting of test messages synchronous. E.g. continuous testing needs
+            // to know when a tests starts/finishes. If we report that asynchronously, it can't
+            // accurately track what code is affected by which tests
+            if (environment.DisableAllConcurrency)
+                options.SetSynchronousMessageReporting(true);
+
             Logger.LogVerbose("Starting execution");
             LogExecutionOptions(options);
 

resharper/src/runner/Tasks/AttributeNames.cs

@@ -9,6 +9,7 @@ public static class AttributeNames
         public const string Explicitly = "Explicitly";
         public const string Dynamic = "Dynamic";
         public const string ProjectId = "ProjectId";
+        public const string DisableAllConcurrency = "DisableAllConcurrency";
 
         // Note the case. TestRemoteChannelMessageListener strips this attribute
         // when running tests. Makes for cleaner gold output

resharper/src/runner/Tasks/XunitBootstrapTask.cs

@@ -1,41 +1,54 @@
 using System;
+using System.Globalization;
 using System.Xml;
 using JetBrains.ReSharper.TaskRunnerFramework;
 
 namespace XunitContrib.Runner.ReSharper.RemoteRunner.Tasks
 {
-    // This class's only purpose is to be the first class in a task sequence.
-    // The first task is always serialised and sent from the external process
-    // to the main ReSharper process. If we use XunitTestAssemblyTask as the
-    // first class, it includes the path to the assembly under test, which
-    // means failing tests if the assembly is in a different location to what's
-    // in the gold files. So, this class is used first, and when it gets
-    // serialised, there is nothing changeable in it (the project id is constant
-    // for tests) and as long as we never report the assembly task (there's no
-    // need, it's not a meaningful task - not associated with an element), then
-    // our tests will pass
+    // This class serves two purposes. Firstly, it notifies the runner of some
+    // config that's only available from the provider (i.e. if the host is running,
+    // debugging, covering or continuous testing). Secondly, it means that the
+    // first task in the sequence isn't XunitTestAssemblyTask, which contains
+    // the path of the assembly under test. This greatly helps the tests for
+    // the runner itself, as the first task is output to the gold file, and
+    // showing the path to the assembly under test will break things if we
+    // run the tests in a different location. We can serialise this task,
+    // as the project ID is constant in test runs, and there is nothing
+    // else changeable in it. As long as we never report the assembly
+    // task (there's no need, it's not a meaningful task - not associated
+    // with an element), then our tests will pass.
     [Serializable]
     public class XunitBootstrapTask : RemoteTask, IEquatable<XunitBootstrapTask>
     {
-        public XunitBootstrapTask(string projectId)
+
+        public XunitBootstrapTask(string projectId, bool disableAllConcurrency)
             : base(XunitTaskRunner.RunnerId)
         {
+            DisableAllConcurrency = disableAllConcurrency;
             ProjectId = projectId;
         }
 
         public XunitBootstrapTask(XmlElement element)
             : base(element)
         {
             ProjectId = GetXmlAttribute(element, AttributeNames.ProjectId);
+
+            bool disableAllConcurrency;
+            if (!bool.TryParse(GetXmlAttribute(element, AttributeNames.DisableAllConcurrency), out disableAllConcurrency))
+                disableAllConcurrency = false;
+            DisableAllConcurrency = disableAllConcurrency;
         }
 
         public string ProjectId { get; set; }
+        public bool DisableAllConcurrency { get; set; }
+
         public override bool IsMeaningfulTask { get { return false; }}
 
         public override void SaveXml(XmlElement element)
         {
             base.SaveXml(element);
             SetXmlAttribute(element, AttributeNames.ProjectId, ProjectId);
+            SetXmlAttribute(element, AttributeNames.DisableAllConcurrency, DisableAllConcurrency.ToString(CultureInfo.InvariantCulture));
         }
 
         public override bool Equals(object obj)

resharper/src/runner/TestEnvironment.cs

@@ -9,7 +9,7 @@ namespace XunitContrib.Runner.ReSharper.RemoteRunner
 {
     public class TestEnvironment
     {
-        public TestEnvironment(XunitTestAssemblyTask assemblyTask)
+        public TestEnvironment(XunitTestAssemblyTask assemblyTask, bool disableAllConcurrency)
         {
             // Use the assembly in the folder that the user has specified, or, if not, use the assembly location
             var assemblyFolder = GetAssemblyFolder(ResharperConfiguration, assemblyTask);
@@ -24,6 +24,9 @@ public TestEnvironment(XunitTestAssemblyTask assemblyTask)
             // clean up at the end of the run
             ShadowCopyPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
 
+            // The hosts require disabling concurrency (e.g. code coverage, continuous testing, dotMemoryUnit)
+            DisableAllConcurrency = disableAllConcurrency;
+
             TestAssemblyConfiguration = GetXunitConfiguration(AssemblyPath, ConfigPath);
             MergeConfiguration();
             DiagnosticMessages = new DiagnosticMessages(TestAssemblyConfiguration.DiagnosticMessagesOrDefault);
@@ -32,6 +35,7 @@ public TestEnvironment(XunitTestAssemblyTask assemblyTask)
         public string AssemblyPath { get; private set; }
         public string ConfigPath { get; private set; }
         public string ShadowCopyPath { get; private set; }
+        public bool DisableAllConcurrency { get; private set; }
         public DiagnosticMessages DiagnosticMessages { get; private set; }
         public TestAssemblyConfiguration TestAssemblyConfiguration { get; private set; }
 
@@ -90,6 +94,14 @@ private void MergeConfiguration()
             // I can tell if the xunit config was set, but not if the ReSharper one isn't. Which takes precedence?
             // (ReSharper's for now) Add diagnostic warning that the xunit
             TestAssemblyConfiguration.ShadowCopy = ResharperConfiguration.ShadowCopy;
+
+            // Disable parallelisation if the test host requests it (e.g. for code coverage, continuous testing
+            // or dotMemoryUnit). Note that we'll also need to disable async test message reporting
+            if (DisableAllConcurrency)
+            {
+                TestAssemblyConfiguration.ParallelizeAssembly = false;
+                TestAssemblyConfiguration.ParallelizeTestCollections = false;
+            }
         }
     }
 }

resharper/src/runner/TestRunner.cs

@@ -19,9 +19,9 @@ public TestRunner(IRemoteTaskServer server)
 
         public RunContext RunContext { get; private set; }
 
-        public void Run(XunitTestAssemblyTask assemblyTask)
+        public void Run(XunitTestAssemblyTask assemblyTask, bool disableAllConcurrency)
         {
-            var environment = new TestEnvironment(assemblyTask);
+            var environment = new TestEnvironment(assemblyTask, disableAllConcurrency);
             try
             {
                 if (environment.TestAssemblyConfiguration.ShadowCopyOrDefault)

resharper/src/runner/XunitTaskRunner.cs

@@ -17,13 +17,14 @@ public XunitTaskRunner(IRemoteTaskServer server)
 
         public override void ExecuteRecursive(TaskExecutionNode node)
         {
-            // node is XunitBootstapTask, which we don't care about (see comment on XunitBootstrapTask)
+            var bootstrapTask = (XunitBootstrapTask) node.RemoteTask;
+
             var assemblyTaskNode = node.Children[0];
             var assemblyTask = (XunitTestAssemblyTask) assemblyTaskNode.RemoteTask;
 
             PopulateRunContext(testRunner.RunContext, assemblyTaskNode);
 
-            testRunner.Run(assemblyTask);
+            testRunner.Run(assemblyTask, bootstrapTask.DisableAllConcurrency);
         }
 
         public override void Abort()