From 0086d4eaa09b6cf8c2543512f9057645b7a9b166 Mon Sep 17 00:00:00 2001 From: Johnny Hendriks Date: Mon, 17 Sep 2018 08:39:56 +0200 Subject: [PATCH] Improves how is dealt with problematic test case names. --- Docs/CHANGELOG.md | 1 + Docs/Capabilities.md | 4 +- Libraries/Catch2Interface/Executor.cs | 2 +- Libraries/Catch2Interface/TestResult.cs | 65 +++++++++++----- Libraries/Catch2TestAdapter/TestExecutor.cs | 75 ++++++++++-------- UnitTests/UT_Catch2Interface/ExecutorTests.cs | 78 ++++++++++--------- .../TestStrings_TestResult.Designer.cs | 8 +- .../Resources/TestStrings_TestResult.resx | 8 +- .../UT_Catch2Interface/TestResultTests.cs | 18 ++--- .../source.extension.vsixmanifest | 2 +- 10 files changed, 152 insertions(+), 109 deletions(-) diff --git a/Docs/CHANGELOG.md b/Docs/CHANGELOG.md index a56f221..a4b6488 100644 --- a/Docs/CHANGELOG.md +++ b/Docs/CHANGELOG.md @@ -7,6 +7,7 @@ Changes are relative to v1.0.0 ### Extended Features - Improve Additional info output with respect to section. Is now more like standard Catch2 output. +- Improve how is dealt with problematic test case names. Tests with such names are now marked as skipped, with an error message that suggests a possible solution. ### Bug fixes diff --git a/Docs/Capabilities.md b/Docs/Capabilities.md index 7014c09..478281b 100644 --- a/Docs/Capabilities.md +++ b/Docs/Capabilities.md @@ -31,10 +31,12 @@ Here `` may have several "." delimiters. This scheme helps me to keep ### Special characters -Be aware that trailing spaces are automatically removed from a test case name. Also, names ending in a backslash ("\\") cannot be called specifically by the **Test Adapter for Catch2**. As of v1.3.0 a workaround is used where basically the backslashes are stripped from the end of the name. Subsequently all tests that start with the remaining test name are called. So there is a chance more than one test case will be called. This is handled in the same way as test case names that differ only in case (see [below](#catch2-specific)). +Be aware that names ending in a backslash ("\\") cannot be called specifically by the **Test Adapter for Catch2**. As of v1.3.0 a workaround is used where basically the backslashes are stripped from the end of the name. Subsequently all tests that start with the remaining test name are called. So there is a chance more than one test case will be called. This is handled in the same way as test case names that differ only in case (see [below](#catch2-specific)). If you want to call a specific test case from the command line you need to escape any comma, double quote, open square bracket and backslash characters (_i.e._, use "\\,", "\\"", "\\[" and "\\\\"). This is basically what the **Test Adapter for Catch2** does internally when it calls a test. Otherwise any printable ASCII character can be safely used in a test case name. No guarantees are given for the use of other characters (_e.g._, UTF-8). +During test case name discovery trailing spaces are automatically removed from a test case name. Consequently, test cases with names that end in a space character cannot be specifically run by the **Test Adapter for Catch2**. + ### Catch2 specific Test case names are semi case sensitive. Test cases which only differ in case are separate test cases as far as Catch2 is concerned. However, they cannot be called individually via the command line. As such, when such a test is called, all tests with the same case insensitive name are run. As of version 1.3 of the **Test Adapter for Catch2** this situation is handled more gracefully and at least the correct test results is shown for the test case. The assertion statistics shown for the test case are those of all the run tests, including the ones with only differ in case. As a result, it is possible that a passed test case contains failed assertions in the shown assertion statistics. For clarity a note is added to the test case result message to indicate this. Before version 1.3 of the **Test Adapter for Catch2** this corner case was not handled well and therefore could report the wrong result for those test cases. \ No newline at end of file diff --git a/Libraries/Catch2Interface/Executor.cs b/Libraries/Catch2Interface/Executor.cs index 686c11b..1d9cd82 100644 --- a/Libraries/Catch2Interface/Executor.cs +++ b/Libraries/Catch2Interface/Executor.cs @@ -137,7 +137,7 @@ public TestResult Run(string testname, string source) LogDebug(output.Result); Log = _logbuilder.ToString(); - // Process testrun output (should have result from a single testcase) + // Process testrun output return new TestResult(output.Result, testname, _settings); } } diff --git a/Libraries/Catch2Interface/TestResult.cs b/Libraries/Catch2Interface/TestResult.cs index bef60d7..f3f650d 100644 --- a/Libraries/Catch2Interface/TestResult.cs +++ b/Libraries/Catch2Interface/TestResult.cs @@ -20,6 +20,20 @@ namespace Catch2Interface { +/*YAML +Enum : + Description : > + This enum represents the test result outcomes. +*/ + public enum TestOutcomes + { + Cancelled, + Failed, + Passed, + Skipped, + Timedout + } + /*YAML Class : Description : > @@ -53,7 +67,7 @@ public class TestResult public TestResult() { - Cancelled = true; + Outcome = TestOutcomes.Cancelled; } public TestResult(string xmloutput, string testname, Settings settings) @@ -68,8 +82,8 @@ public TestResult( TimeSpan duration , string msg , string standardout ) { - TimedOut = true; Duration = duration; + Outcome = TestOutcomes.Timedout; StandardOut = standardout; } #endregion // Costructor @@ -87,10 +101,7 @@ public TestResult( TimeSpan duration public string StandardOut { get; private set; } public string StandardError { get; private set; } - public bool Cancelled { get; private set; } = false; - public bool TimedOut { get; private set; } = false; - - public bool Success { get; private set; } = false; + public TestOutcomes Outcome { get; private set; } #endregion // Properties @@ -365,25 +376,39 @@ void ExtractTestResult(XmlNode nodeGroup) var nodesTestCases = nodeGroup.SelectNodes("TestCase"); _testcasecount = nodesTestCases.Count; - foreach (XmlNode nodeTestCase in nodesTestCases) + + if(_testcasecount == 0) { - var testcase = new Reporter.TestCase(nodeTestCase); - if(_testcasecount != 1 && testcase.Name != _testname) continue; + // Special case. It appears the used test case name could not be found by Catch2. + // As such the test case is effectively skipped. + // This is an edge case that can typically be resolved by changing the test case name. + // So tell the user about it. + Outcome = TestOutcomes.Skipped; + ErrorMessage = $"Testcase could not be run. Probably the used testcase name is the cause. Change the testcase name and try again. Typically, this problem is encountered when the last character of the testcase name is a space."; + return; + } + else + { + foreach (XmlNode nodeTestCase in nodesTestCases) + { + var testcase = new Reporter.TestCase(nodeTestCase); + if(_testcasecount != 1 && testcase.Name != _testname) continue; - _testcase = testcase; + _testcase = testcase; - Success = _testcase.OverallResult.Success; - Duration = _testcase.OverallResult.Duration; - StandardOut = _testcase.OverallResult.StdOut; - StandardError = _testcase.OverallResult.StdErr; + Outcome = _testcase.OverallResult.Success ? TestOutcomes.Passed : TestOutcomes.Failed; + Duration = _testcase.OverallResult.Duration; + StandardOut = _testcase.OverallResult.StdOut; + StandardError = _testcase.OverallResult.StdErr; - ExtractMessages(); + ExtractMessages(); - // Statistics - ExtractOverallResults(nodeGroup); + // Statistics + ExtractOverallResults(nodeGroup); - GenerateMessages(); - return; + GenerateMessages(); + return; + } } SetInvalidTestRunnerOutput(); @@ -475,7 +500,7 @@ private string GenerateAssertionInfo() } private void SetInvalidTestRunnerOutput() { - Success = false; + Outcome = TestOutcomes.Failed; OverallResults = new Reporter.OverallResults(); ErrorMessage = $"Invalid test runner output.{Environment.NewLine}" + $"-------------------------------------------------------------------------------{Environment.NewLine}" diff --git a/Libraries/Catch2TestAdapter/TestExecutor.cs b/Libraries/Catch2TestAdapter/TestExecutor.cs index e8e43c1..a81bd47 100644 --- a/Libraries/Catch2TestAdapter/TestExecutor.cs +++ b/Libraries/Catch2TestAdapter/TestExecutor.cs @@ -201,39 +201,50 @@ private TestResult RunTest(TestCase test) } // Process test results - if( testresult.TimedOut ) + switch (testresult.Outcome) { - LogVerbose(TestMessageLevel.Warning, "Time out"); - result.Outcome = TestOutcome.Skipped; - result.ErrorMessage = testresult.ErrorMessage; - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut)); - result.Duration = testresult.Duration; - } - else if( testresult.Cancelled ) - { - result.Outcome = TestOutcome.None; - } - else - { - result.Outcome = testresult.Success ? TestOutcome.Passed : TestOutcome.Failed; - result.Duration = testresult.Duration; - result.ErrorMessage = testresult.ErrorMessage; - result.ErrorStackTrace = testresult.ErrorStackTrace; - - if( !string.IsNullOrEmpty(testresult.StandardOut) ) - { - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut )); - } - - if( !string.IsNullOrEmpty(testresult.StandardError) ) - { - result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, testresult.StandardError )); - } - - if( !string.IsNullOrEmpty(testresult.AdditionalInfo) ) - { - result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, testresult.AdditionalInfo )); - } + case Catch2Interface.TestOutcomes.Timedout: + LogVerbose(TestMessageLevel.Warning, "Time out"); + result.Outcome = TestOutcome.Skipped; + result.ErrorMessage = testresult.ErrorMessage; + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut)); + result.Duration = testresult.Duration; + break; + case Catch2Interface.TestOutcomes.Cancelled: + result.Outcome = TestOutcome.None; + break; + case Catch2Interface.TestOutcomes.Skipped: + result.Outcome = TestOutcome.Skipped; + result.ErrorMessage = testresult.ErrorMessage; + break; + default: + if( testresult.Outcome == Catch2Interface.TestOutcomes.Passed) + { + result.Outcome = TestOutcome.Passed; + } + else + { + result.Outcome = TestOutcome.Failed; + } + result.Duration = testresult.Duration; + result.ErrorMessage = testresult.ErrorMessage; + result.ErrorStackTrace = testresult.ErrorStackTrace; + + if (!string.IsNullOrEmpty(testresult.StandardOut)) + { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut)); + } + + if (!string.IsNullOrEmpty(testresult.StandardError)) + { + result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, testresult.StandardError)); + } + + if (!string.IsNullOrEmpty(testresult.AdditionalInfo)) + { + result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, testresult.AdditionalInfo)); + } + break; } } diff --git a/UnitTests/UT_Catch2Interface/ExecutorTests.cs b/UnitTests/UT_Catch2Interface/ExecutorTests.cs index f6b0d25..e7199fd 100644 --- a/UnitTests/UT_Catch2Interface/ExecutorTests.cs +++ b/UnitTests/UT_Catch2Interface/ExecutorTests.cs @@ -86,16 +86,14 @@ public void TestRun_Basic() var executor = new Executor(settings, _pathSolution, _pathTestRun); var result = executor.Run("Testset03.Tests01. 01p Basic", Path_Testset03); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(4, result.OverallResults.Successes); Assert.AreEqual(4, result.OverallResults.TotalAssertions); result = executor.Run("Testset03.Tests01. 01f Basic", Path_Testset03); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(2, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); Assert.AreEqual(3, result.OverallResults.TotalAssertions); @@ -108,32 +106,28 @@ public void TestRun_TestNames_Pass() var executor = new Executor(settings, _pathSolution, _pathTestRun); var result = executor.Run("Testset02.Tests01. abcdefghijklmnopqrstuvwxyz", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); result = executor.Run("Testset02.Tests01. ZXYWVUTSRQPONMLKJIHGFEDCBA", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); result = executor.Run("Testset02.Tests01. 0123456789", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); + Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); result = executor.Run(@"Testset02.Tests01. []{}!@#$%^&*()_-+=|\?/><,~`';:", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); result = executor.Run("Testset02.Tests01. \"name\"", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); @@ -141,17 +135,25 @@ public void TestRun_TestNames_Pass() // cannot be called specifically via the Catch2 commandline. // Needs to be called in a special way. result = executor.Run(@"Testset02.Tests01. \", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); - Assert.AreEqual(6, result.OverallResults.Successes); + Assert.AreEqual(8, result.OverallResults.Successes); // No issues with '\' elsewhere in the name/ result = executor.Run(@"\Testset02.Tests01. name", Path_Testset02); - Assert.IsTrue(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.AreEqual(0, result.OverallResults.Failures); Assert.AreEqual(1, result.OverallResults.Successes); + + // Another special case, test names that end in spaces. + // The spaces are lost in discovery resulting in the testcase not being found. + result = executor.Run(@"Testset02.Tests02. End with space", Path_Testset02); + Assert.AreEqual(result.Outcome, TestOutcomes.Skipped); + Assert.AreEqual(new TimeSpan(0), result.Duration); + + result = executor.Run(@"Testset02.Tests02. End with spaces", Path_Testset02); + Assert.AreEqual(result.Outcome, TestOutcomes.Skipped); + Assert.AreEqual(new TimeSpan(0), result.Duration); } [TestMethod] @@ -159,34 +161,29 @@ public void TestRun_TestNames_Fail() { var settings = new Settings(); var executor = new Executor(settings, _pathSolution, _pathTestRun); - + var result = executor.Run("Testset02.Tests02. abcdefghijklmnopqrstuvwxyz", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); result = executor.Run("Testset02.Tests02. ZXYWVUTSRQPONMLKJIHGFEDCBA", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); result = executor.Run("Testset02.Tests02. 0123456789", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); result = executor.Run(@"Testset02.Tests02. []{}!@#$%^&*()_-+=|\?/><,~`';:", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); result = executor.Run("Testset02.Tests02. \"name\"", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); @@ -194,17 +191,25 @@ public void TestRun_TestNames_Fail() // cannot be called specifically via the Catch2 commandline. // Needs to be called in a special way. result = executor.Run(@"Testset02.Tests02. \", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); - Assert.AreEqual(6, result.OverallResults.Failures); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); + Assert.AreEqual(8, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); // No issues with '\' elsewhere in the name/ result = executor.Run(@"\Testset02.Tests02. name", Path_Testset02); - Assert.IsFalse(result.Success); - Assert.IsFalse(result.TimedOut); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.AreEqual(1, result.OverallResults.Failures); Assert.AreEqual(0, result.OverallResults.Successes); + + // Another special case, test names that end in spaces. + // The spaces are lost in discovery resulting in the testcase not being found. + result = executor.Run(@"Testset02.Tests02. End with space", Path_Testset02); + Assert.AreEqual(TestOutcomes.Skipped, result.Outcome); + Assert.AreEqual(new TimeSpan(0), result.Duration); + + result = executor.Run(@"Testset02.Tests02. End with spaces", Path_Testset02); + Assert.AreEqual(TestOutcomes.Skipped, result.Outcome); + Assert.AreEqual(new TimeSpan(0), result.Duration); } [TestMethod] @@ -215,8 +220,7 @@ public void TestRun_Timeout() var executor = new Executor(settings, _pathSolution, _pathTestRun); var result = executor.Run("Wait forever", Path_Dummy); - Assert.IsFalse(result.Success); - Assert.IsTrue(result.TimedOut); + Assert.AreEqual(result.Outcome, TestOutcomes.Timedout); Assert.AreEqual(new TimeSpan(0,0,0,0,2000), result.Duration); } diff --git a/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.Designer.cs b/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.Designer.cs index 6515f32..9e7e960 100644 --- a/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.Designer.cs +++ b/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.Designer.cs @@ -94,12 +94,12 @@ internal static string Invalid { /// /// Looks up a localized string similar to <?xml version="1.0" encoding="UTF-8"?> ///<Catch name="QDummy.exe"> - /// <Group name="QDummy.exe"> - /// <CaseTest name="Dummy" filename="d:\dummy.cpp" line="64"> + /// <BadGroup name="QDummy.exe"> + /// <TestCase name="Dummy" filename="d:\dummy.cpp" line="64"> /// <OverallResult success="true" durationInSeconds="0.000981"/> - /// </CaseTest> + /// </TestCase> /// <OverallResults successes="6" failures="0" expectedFailures="0"/> - /// </Group> + /// </BadGroup> /// <OverallResults successes="6" failures="1" expectedFailures="0"/> ///</Catch>. /// diff --git a/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.resx b/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.resx index ae693ac..3b37ad9 100644 --- a/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.resx +++ b/UnitTests/UT_Catch2Interface/Resources/TestStrings_TestResult.resx @@ -147,12 +147,12 @@ It is invalid input. <?xml version="1.0" encoding="UTF-8"?> <Catch name="QDummy.exe"> - <Group name="QDummy.exe"> - <CaseTest name="Dummy" filename="d:\dummy.cpp" line="64"> + <BadGroup name="QDummy.exe"> + <TestCase name="Dummy" filename="d:\dummy.cpp" line="64"> <OverallResult success="true" durationInSeconds="0.000981"/> - </CaseTest> + </TestCase> <OverallResults successes="6" failures="0" expectedFailures="0"/> - </Group> + </BadGroup> <OverallResults successes="6" failures="1" expectedFailures="0"/> </Catch> diff --git a/UnitTests/UT_Catch2Interface/TestResultTests.cs b/UnitTests/UT_Catch2Interface/TestResultTests.cs index 3b261d1..abbb020 100644 --- a/UnitTests/UT_Catch2Interface/TestResultTests.cs +++ b/UnitTests/UT_Catch2Interface/TestResultTests.cs @@ -20,7 +20,7 @@ public void TestIncomplete() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.Incomplete, "Dummy", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsTrue(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -30,7 +30,7 @@ public void TestInvalid() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.Invalid, "Dummy", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsTrue(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -40,7 +40,7 @@ public void TestInvalidXml() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.InvalidXml, "Dummy", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsTrue(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -50,7 +50,7 @@ public void TestMultipleTestCases_test1() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.MultipleTestCases, "dummy", settings); - Assert.IsTrue(result.Success); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.IsFalse(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -60,7 +60,7 @@ public void TestMultipleTestCases_test2() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.MultipleTestCases, "Dummy", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsFalse(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -70,7 +70,7 @@ public void TestMultipleTestCases_test3() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.MultipleTestCases, "DUMMY", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsTrue(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -80,7 +80,7 @@ public void TestPostXmlText() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.PostXmlText, "Dummy", settings); - Assert.IsFalse(result.Success); + Assert.AreEqual(TestOutcomes.Failed, result.Outcome); Assert.IsFalse(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -91,7 +91,7 @@ public void TestSingleTestCase_test1() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.SingleTestCase, "Dummy", settings); - Assert.IsTrue(result.Success); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.IsFalse(result.ErrorMessage.Contains("Invalid test runner output.")); } @@ -101,7 +101,7 @@ public void TestSingleTestCase_test2() var settings = new Settings(); var result = new Catch2Interface.TestResult(Resources.TestStrings_TestResult.SingleTestCase, "OtherName", settings); - Assert.IsTrue(result.Success); + Assert.AreEqual(TestOutcomes.Passed, result.Outcome); Assert.IsFalse(result.ErrorMessage.Contains("Invalid test runner output.")); } } diff --git a/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest b/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest index e2f40e5..55d1240 100644 --- a/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest +++ b/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest @@ -1,7 +1,7 @@ - + Test Adapter for Catch2 Test Adapter for use with the Catch2 C++ unit test framework. https://github.com/JohnnyHendriks/TestAdapter_Catch2