diff --git a/src/TestEngine/testcentric.engine.core/Internal/XmlHelper.cs b/src/TestEngine/testcentric.engine.core/Internal/XmlHelper.cs index 48d17004..5e366150 100644 --- a/src/TestEngine/testcentric.engine.core/Internal/XmlHelper.cs +++ b/src/TestEngine/testcentric.engine.core/Internal/XmlHelper.cs @@ -40,11 +40,12 @@ public static XmlNode CreateXmlNode(string xml) /// The node to which the attribute should be added. /// The name of the attribute. /// The value of the attribute. - public static void AddAttribute(this XmlNode node, string name, string value) + public static XmlNode AddAttribute(this XmlNode node, string name, string value) { XmlAttribute attr = node.OwnerDocument.CreateAttribute(name); attr.Value = value; node.Attributes.Append(attr); + return node; } /// diff --git a/src/TestEngine/testcentric.engine/Internal/ResultHelper.cs b/src/TestEngine/testcentric.engine/Internal/ResultHelper.cs index a60faee8..60c566df 100644 --- a/src/TestEngine/testcentric.engine/Internal/ResultHelper.cs +++ b/src/TestEngine/testcentric.engine/Internal/ResultHelper.cs @@ -61,6 +61,71 @@ public static TestEngineResult MakeProjectResult(this TestEngineResult result, T return Aggregate(result, TEST_SUITE_ELEMENT, PROJECT_SUITE_TYPE, package.ID, package.Name, package.FullName); } + // The TestEngineResult returned to MasterTestRunner contains no info + // about projects. At this point, if there are any projects, the result + // needs to be modified to include info about them. Doing it this way + // allows the lower-level runners to be completely ignorant of projects + public static TestEngineResult AddProjectPackages(this TestEngineResult result, TestPackage testPackage) + { + if (result == null) throw new ArgumentNullException("result"); + + // See if we have any projects to deal with. At this point, + // any subpackage, which itself has subpackages, is a project + // we expanded. + bool hasProjects = false; + foreach (var p in testPackage.SubPackages) + hasProjects |= p.HasSubPackages(); + + // If no Projects, there's nothing to do + if (!hasProjects) + return result; + + // If there is just one subpackage, it has to be a project and we don't + // need to rebuild the XML but only wrap it with a project result. + if (testPackage.SubPackages.Count == 1) + return result.MakeProjectResult(testPackage.SubPackages[0]); + + // Most complex case - we need to work with the XML in order to + // examine and rebuild the result to include project nodes. + // NOTE: The algorithm used here relies on the ordering of nodes in the + // result matching the ordering of subpackages under the top-level package. + // If that should change in the future, then we would need to implement + // identification and summarization of projects into each of the lower- + // level TestEngineRunners. In that case, we will be warned by failures + // of some of the MasterTestRunnerTests. + + // Start a fresh TestEngineResult for top level + var topLevelResult = new TestEngineResult(); + int nextTest = 0; + + foreach (var subPackage in testPackage.SubPackages) + { + if (subPackage.HasSubPackages()) + { + // This is a project, create an intermediate result + var projectResult = new TestEngineResult(); + + // Now move any children of this project under it. As noted + // above, we must rely on ordering here because (1) the + // fullname attribute is not reliable on all nunit framework + // versions, (2) we may have duplicates of the same assembly + // and (3) we have no info about the id of each assembly. + int numChildren = subPackage.SubPackages.Count; + while (numChildren-- > 0) + projectResult.Add(result.XmlNodes[nextTest++]); + + topLevelResult.Add(projectResult.MakeProjectResult(subPackage).Xml); + } + else + { + // Add the next assembly package to our new result + topLevelResult.Add(result.XmlNodes[nextTest++]); + } + } + + return topLevelResult; + } + /// /// Aggregate all the separate assembly results of a test run as a single node. /// @@ -69,7 +134,33 @@ public static TestEngineResult MakeProjectResult(this TestEngineResult result, T /// A TestEngineResult with a single top-level element. public static TestEngineResult MakeTestRunResult(this TestEngineResult result, TestPackage package) { - return Aggregate(result, TEST_RUN_ELEMENT, package.ID, package.Name, package.FullName); + return Aggregate(result.AddProjectPackages(package), TEST_RUN_ELEMENT, package.ID, package.Name, package.FullName); + } + + public static TestEngineResult InsertFilterElement(this TestEngineResult result, TestFilter filter) + { + // Convert the filter to an XmlNode + var tempNode = XmlHelper.CreateXmlNode(filter.Text); + + // Don't include it if it's an empty filter + if (tempNode.ChildNodes.Count > 0) + { + var doc = result.Xml.OwnerDocument; + if (doc != null) + { + var filterElement = doc.ImportNode(tempNode, true); + result.Xml.InsertAfter(filterElement, null); + } + } + + return result; + } + + public static TestEngineResult InsertCommandLineElement(this TestEngineResult result, string commandLine) + { + result.Xml.AddElementWithCDataSection("command-line", commandLine); + + return result; } /// diff --git a/src/TestEngine/testcentric.engine/Runners/MasterTestRunner.cs b/src/TestEngine/testcentric.engine/Runners/MasterTestRunner.cs index e9f2f307..5e58e74a 100644 --- a/src/TestEngine/testcentric.engine/Runners/MasterTestRunner.cs +++ b/src/TestEngine/testcentric.engine/Runners/MasterTestRunner.cs @@ -9,6 +9,7 @@ using System.Globalization; using System.IO; using System.Reflection; +using System.Threading.Tasks; using System.Xml; using NUnit.Engine; using TestCentric.Engine.Internal; @@ -103,7 +104,8 @@ protected bool IsPackageLoaded /// An XmlNode representing the loaded assembly. public XmlNode Load() { - LoadResult = PrepareResult(GetEngineRunner().Load()).MakeTestRunResult(TestPackage); + LoadResult = GetEngineRunner().Load() + .MakeTestRunResult(TestPackage); return LoadResult.Xml; } @@ -124,7 +126,8 @@ public void Unload() /// If no package has been loaded public XmlNode Reload() { - LoadResult = PrepareResult(GetEngineRunner().Reload()).MakeTestRunResult(TestPackage); + LoadResult = GetEngineRunner().Reload() + .MakeTestRunResult(TestPackage); return LoadResult.Xml; } @@ -177,11 +180,13 @@ public void StopRun(bool force) // When running under .NET Core, the test framework will not be able to kill the // threads currently running tests. We handle cleanup here in case that happens. - if (force && !_eventDispatcher.WaitForCompletion(WAIT_FOR_CANCEL_TO_COMPLETE)) + if (force) { // Send completion events for any tests, which were still running _eventDispatcher.IssuePendingNotifications(); + IsTestRunning = false; + // Signal completion of the run _eventDispatcher.DispatchEvent($""); } @@ -195,7 +200,7 @@ public void StopRun(bool force) /// An XmlNode representing the tests found. public XmlNode Explore(TestFilter filter) { - LoadResult = PrepareResult(GetEngineRunner().Explore(filter)) + LoadResult = GetEngineRunner().Explore(filter) .MakeTestRunResult(TestPackage); return LoadResult.Xml; @@ -247,71 +252,6 @@ internal ITestEngineRunner GetEngineRunner() return _engineRunner; } - // The TestEngineResult returned to MasterTestRunner contains no info - // about projects. At this point, if there are any projects, the result - // needs to be modified to include info about them. Doing it this way - // allows the lower-level runners to be completely ignorant of projects - private TestEngineResult PrepareResult(TestEngineResult result) - { - if (result == null) throw new ArgumentNullException("result"); - - // See if we have any projects to deal with. At this point, - // any subpackage, which itself has subpackages, is a project - // we expanded. - bool hasProjects = false; - foreach (var p in TestPackage.SubPackages) - hasProjects |= p.HasSubPackages(); - - // If no Projects, there's nothing to do - if (!hasProjects) - return result; - - // If there is just one subpackage, it has to be a project and we don't - // need to rebuild the XML but only wrap it with a project result. - if (TestPackage.SubPackages.Count == 1) - return result.MakeProjectResult(TestPackage.SubPackages[0]); - - // Most complex case - we need to work with the XML in order to - // examine and rebuild the result to include project nodes. - // NOTE: The algorithm used here relies on the ordering of nodes in the - // result matching the ordering of subpackages under the top-level package. - // If that should change in the future, then we would need to implement - // identification and summarization of projects into each of the lower- - // level TestEngineRunners. In that case, we will be warned by failures - // of some of the MasterTestRunnerTests. - - // Start a fresh TestEngineResult for top level - var topLevelResult = new TestEngineResult(); - int nextTest = 0; - - foreach (var subPackage in TestPackage.SubPackages) - { - if (subPackage.HasSubPackages()) - { - // This is a project, create an intermediate result - var projectResult = new TestEngineResult(); - - // Now move any children of this project under it. As noted - // above, we must rely on ordering here because (1) the - // fullname attribute is not reliable on all nunit framework - // versions, (2) we may have duplicates of the same assembly - // and (3) we have no info about the id of each assembly. - int numChildren = subPackage.SubPackages.Count; - while (numChildren-- > 0) - projectResult.Add(result.XmlNodes[nextTest++]); - - topLevelResult.Add(projectResult.MakeProjectResult(subPackage).Xml); - } - else - { - // Add the next assembly package to our new result - topLevelResult.Add(result.XmlNodes[nextTest++]); - } - } - - return topLevelResult; - } - /// /// Unload any loaded TestPackage. /// @@ -345,7 +285,7 @@ private int CountTests(TestFilter filter) /// A TestEngineResult giving the result of the test execution private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter) { - _eventDispatcher.ClearListeners(); + _eventDispatcher.InitializeForRun(); if (listener != null) _eventDispatcher.Listeners.Add(listener); @@ -362,22 +302,21 @@ private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter try { - var startRunNode = XmlHelper.CreateTopLevelElement("start-run"); - startRunNode.AddAttribute("count", CountTests(filter).ToString()); - startRunNode.AddAttribute("start-time", XmlConvert.ToString(startTime, "u")); - startRunNode.AddAttribute("engine-version", engineVersion); - startRunNode.AddAttribute("clr-version", clrVersion); + var startRunNode = XmlHelper.CreateTopLevelElement("start-run") + .AddAttribute("count", CountTests(filter).ToString()) + .AddAttribute("start-time", XmlConvert.ToString(startTime, "u")) + .AddAttribute("engine-version", engineVersion) + .AddAttribute("clr-version", clrVersion); - InsertCommandLineElement(startRunNode); + startRunNode.AddElementWithCDataSection("command-line", Environment.CommandLine); _eventDispatcher.OnTestEvent(startRunNode.OuterXml); - TestEngineResult result = PrepareResult(GetEngineRunner().Run(_eventDispatcher, filter)).MakeTestRunResult(TestPackage); - - // These are inserted in reverse order, since each is added as the first child. - InsertFilterElement(result.Xml, filter); - - InsertCommandLineElement(result.Xml); + // Insertions are done in reverse order, since each is added as the first child. + TestEngineResult result = GetEngineRunner().Run(_eventDispatcher, filter) + .MakeTestRunResult(TestPackage) + .InsertFilterElement(filter) + .InsertCommandLineElement(Environment.CommandLine); result.Xml.AddAttribute("engine-version", engineVersion); result.Xml.AddAttribute("clr-version", clrVersion); @@ -396,25 +335,27 @@ private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter { IsTestRunning = false; - var resultXml = XmlHelper.CreateTopLevelElement("test-run"); - resultXml.AddAttribute("id", TestPackage.ID); - resultXml.AddAttribute("result", "Failed"); - resultXml.AddAttribute("label", "Error"); - resultXml.AddAttribute("engine-version", engineVersion); - resultXml.AddAttribute("clr-version", clrVersion); + var result = CreateErrorResult(TestPackage); + result.Xml.AddAttribute("engine-version", engineVersion); + result.Xml.AddAttribute("clr-version", clrVersion); double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency; - resultXml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u")); - resultXml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u")); - resultXml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo)); + result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u")); + result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u")); + result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo)); - _eventDispatcher.OnTestEvent(resultXml.OuterXml); + _eventDispatcher.OnTestEvent(result.Xml.OuterXml); _eventDispatcher.OnTestEvent($""); - return new TestEngineResult(resultXml); + return result; } } + private TestEngineResult CreateErrorResult(TestPackage package) + { + return new TestEngineResult($""); + } + private AsyncTestEngineResult RunTestsAsync(ITestEventListener listener, TestFilter filter) { var testRun = new AsyncTestEngineResult(); @@ -432,42 +373,5 @@ private AsyncTestEngineResult RunTestsAsync(ITestEventListener listener, TestFil return testRun; } - - private static void InsertCommandLineElement(XmlNode resultNode) - { - var doc = resultNode.OwnerDocument; - - if (doc == null) - { - return; - } - - XmlNode cmd = doc.CreateElement("command-line"); - resultNode.InsertAfter(cmd, null); - - var cdata = doc.CreateCDataSection(Environment.CommandLine); - cmd.AppendChild(cdata); - } - - private static void InsertFilterElement(XmlNode resultNode, TestFilter filter) - { - // Convert the filter to an XmlNode - var tempNode = XmlHelper.CreateXmlNode(filter.Text); - - // Don't include it if it's an empty filter - if (tempNode.ChildNodes.Count <= 0) - { - return; - } - - var doc = resultNode.OwnerDocument; - if (doc == null) - { - return; - } - - var filterElement = doc.ImportNode(tempNode, true); - resultNode.InsertAfter(filterElement, null); - } } } diff --git a/src/TestEngine/testcentric.engine/Services/TestEventDispatcher.cs b/src/TestEngine/testcentric.engine/Services/TestEventDispatcher.cs index 35a06924..614b54dc 100644 --- a/src/TestEngine/testcentric.engine/Services/TestEventDispatcher.cs +++ b/src/TestEngine/testcentric.engine/Services/TestEventDispatcher.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Xml; using NUnit.Engine; using TestCentric.Engine.Internal; @@ -21,6 +22,7 @@ public class TestEventDispatcher : MarshalByRefObject, ITestEventListener, IServ private ExtensionService _extensionService; private List _listenerExtensions = new List(); private WorkItemTracker _workItemTracker = new WorkItemTracker(); + private ManualResetEvent _allItemsComplete = new ManualResetEvent(false); private bool _runCancelled; public TestEventDispatcher() @@ -30,15 +32,16 @@ public TestEventDispatcher() public IList Listeners { get; private set; } - public void ClearListeners() + public void InitializeForRun() { _workItemTracker.Clear(); + _allItemsComplete.Reset(); Listeners = new List(_listenerExtensions); } public bool WaitForCompletion(int millisecondsTimeout) { - return _workItemTracker.WaitForCompletion(millisecondsTimeout); + return _allItemsComplete.WaitOne(millisecondsTimeout); } public void IssuePendingNotifications() @@ -48,6 +51,8 @@ public void IssuePendingNotifications() _runCancelled = true; foreach(XmlNode notification in _workItemTracker.CreateCompletionNotifications()) DispatchEvent(notification.OuterXml); + + _allItemsComplete.Set(); } } @@ -85,6 +90,9 @@ internal void DispatchEvent(string report) case "test-suite": string id = xmlNode.GetAttribute("id"); _workItemTracker.RemoveItem(id); + + if (!_workItemTracker.HasPendingItems) + _allItemsComplete.Set(); break; } } diff --git a/src/TestEngine/testcentric.engine/Services/WorkItemTracker.cs b/src/TestEngine/testcentric.engine/Services/WorkItemTracker.cs index cfca40ec..447b6b0b 100644 --- a/src/TestEngine/testcentric.engine/Services/WorkItemTracker.cs +++ b/src/TestEngine/testcentric.engine/Services/WorkItemTracker.cs @@ -35,17 +35,12 @@ namespace TestCentric.Engine.Services internal class WorkItemTracker { private List _itemsInProcess = new List(); - private ManualResetEvent _allItemsComplete = new ManualResetEvent(false); - + + public bool HasPendingItems => _itemsInProcess.Count > 0; + public void Clear() { _itemsInProcess.Clear(); - _allItemsComplete.Reset(); - } - - public bool WaitForCompletion(int millisecondsTimeout) - { - return _allItemsComplete.WaitOne(millisecondsTimeout); } public IEnumerable CreateCompletionNotifications() @@ -59,8 +54,6 @@ public IEnumerable CreateCompletionNotifications() _itemsInProcess.RemoveAt(count); yield return CreateCompletionNotification(startElement); } - - _allItemsComplete.Set(); } private static XmlNode CreateCompletionNotification(XmlNode startElement) @@ -92,8 +85,6 @@ public void RemoveItem(string id) if (item.GetAttribute("id") == id) { _itemsInProcess.Remove(item); - if (_itemsInProcess.Count == 0) - _allItemsComplete.Set(); return; }