diff --git a/.github/workflows/NUnitConsoleAndEngine.CI.yml b/.github/workflows/NUnitConsoleAndEngine.CI.yml index 5d3d582d2..5eff98572 100644 --- a/.github/workflows/NUnitConsoleAndEngine.CI.yml +++ b/.github/workflows/NUnitConsoleAndEngine.CI.yml @@ -45,7 +45,7 @@ jobs: - name: 🔨 Build, Test, Package and Publish (Cake script decides) run: dotnet cake --target=ContinuousIntegration --configuration=Release #--verbosity=diagnostic - # Upload packages before publishing in case of later failures + # Upload packages - name: 💾 Upload build artifacts if: always() uses: actions/upload-artifact@v7 diff --git a/src/Directory.Build.props b/src/Directory.Build.props index b4fcc8cef..e4175408e 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -22,7 +22,7 @@ true 8.0.0 - 4.0.0-beta.3 + 4.0.0-beta.4 NUnit Software NUnit 4 Runner and Engine diff --git a/src/NUnitCommon/nunit.agent.core.tests/Runners/TestAgentRunnerExceptionTests.cs b/src/NUnitCommon/nunit.agent.core.tests/Runners/TestAgentRunnerExceptionTests.cs index 89db9dc62..b932d0196 100644 --- a/src/NUnitCommon/nunit.agent.core.tests/Runners/TestAgentRunnerExceptionTests.cs +++ b/src/NUnitCommon/nunit.agent.core.tests/Runners/TestAgentRunnerExceptionTests.cs @@ -111,23 +111,11 @@ public void Run_Throws_NUnitEngineException() [Test] public void StopRun_Passes_Along_NUnitEngineException() { - _driver.When(x => x.StopRun(Arg.Any())) + _driver.When(x => x.ForcedStop()) .Do(x => { throw new NUnitEngineException("Message"); }); var ex = Assert.Throws(() => _runner.ForcedStop()); - Assert.That(ex.Message, Is.EqualTo("Message")); - } - - [Test] - public void StopRun_Throws_NUnitEngineException() - { - _driver.When(x => x.StopRun(Arg.Any())) - .Do(x => { throw new ArgumentException("Message"); }); - - var ex = Assert.Throws(() => _runner.ForcedStop()); - Assert.That(ex.InnerException, Is.Not.Null); - Assert.That(ex.InnerException, Is.InstanceOf()); - Assert.That(ex.InnerException.Message, Is.EqualTo("Message")); + Assert.That(ex?.Message, Is.EqualTo("Message")); } } } diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/InvalidAssemblyFrameworkDriver.cs b/src/NUnitCommon/nunit.agent.core/Drivers/InvalidAssemblyFrameworkDriver.cs index f16db989d..ae01a4001 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/InvalidAssemblyFrameworkDriver.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/InvalidAssemblyFrameworkDriver.cs @@ -55,7 +55,11 @@ public string Explore(string filter) return GetLoadResult(); } - public void StopRun(bool force) + public void RequestStop() + { + } + + public void ForcedStop() { } diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/NUnit3DriverFactory.cs b/src/NUnitCommon/nunit.agent.core/Drivers/NUnit3DriverFactory.cs index 0ed280d84..d51c19cae 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/NUnit3DriverFactory.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/NUnit3DriverFactory.cs @@ -18,7 +18,7 @@ public class NUnit3DriverFactory : IDriverFactory /// An AssemblyName referring to the possible test framework. public bool IsSupportedTestFramework(AssemblyName reference) { - return NUNIT_FRAMEWORK.Equals(reference.Name, StringComparison.OrdinalIgnoreCase) && reference.Version?.Major >= 3; + return NUNIT_FRAMEWORK.Equals(reference.Name, StringComparison.OrdinalIgnoreCase) && reference.Version?.Major is 3 or 4; } #if NETFRAMEWORK diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi.cs b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi.cs index c4910e1c2..d3a0a6006 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi.cs @@ -5,6 +5,10 @@ namespace NUnit.Engine.Drivers { + /// + /// Driver API for the NUnit Framework. Provides a common interface to al + /// versions of the framework, in spite of differences in their own API. + /// public interface NUnitFrameworkApi { /// @@ -45,7 +49,17 @@ public interface NUnitFrameworkApi /// /// Cancel the ongoing test run. If no test is running, the call is ignored. /// - /// If true, cancel any ongoing test threads, otherwise wait for them to complete. - void StopRun(bool force); + void RequestStop(); + + /// + /// Force the current test run to stop, killing threads or processes if necessary. + /// + /// + void ForcedStop(); + + /// + /// Gets a flag indicating whether ForcedStop is supported for the framework version in use. + /// + bool ForcedStopSupported { get; } } } diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2009.cs b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2009.cs index ddbb3cf5c..fa908d6e4 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2009.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2009.cs @@ -103,7 +103,11 @@ public string Run(ITestEventListener? listener, string filter) public void RunAsync(Action? callback, string filter) => throw new NotImplementedException(); - public void StopRun(bool force) => ExecuteAction(STOP_RUN_ACTION, force); + public void RequestStop() => ExecuteAction(STOP_RUN_ACTION, false); + + public void ForcedStop() => ExecuteAction(STOP_RUN_ACTION, true); + + public bool ForcedStopSupported => true; public string Explore(string filter) { diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2018.cs b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2018.cs index f366e5e21..e521755fb 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2018.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkApi2018.cs @@ -178,11 +178,18 @@ public void RunAsync(Action? callback, string filter) ExecuteMethod(RUN_ASYNC_METHOD, [typeof(Action), typeof(string)], callback, filter); } - public void StopRun(bool force) + public void RequestStop() { - ExecuteMethod(STOP_RUN_METHOD, force); + ExecuteMethod(STOP_RUN_METHOD, false); } + public void ForcedStop() + { + ExecuteMethod(STOP_RUN_METHOD, true); + } + + public bool ForcedStopSupported => _nunitRef.Version.ShouldNotBeNull().Major is 3 or 4; + public string Explore(string filter) { CheckLoadWasCalled(); diff --git a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkDriver.cs b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkDriver.cs index 874c5afec..53505b659 100644 --- a/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkDriver.cs +++ b/src/NUnitCommon/nunit.agent.core/Drivers/NUnitFrameworkDriver.cs @@ -1,8 +1,10 @@ // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt +using NUnit.Common; using NUnit.Engine.Extensibility; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Reflection; #if NETCOREAPP3_1_OR_GREATER @@ -17,9 +19,11 @@ namespace NUnit.Engine.Drivers /// public class NUnitFrameworkDriver : IFrameworkDriver { - private static readonly Version MINIMUM_NUNIT_VERSION = new(3, 2, 0); + private static readonly Version MINIMUM_NUNIT_VERSION_FOR_2018_API = new(3, 2, 0); private static readonly Logger log = InternalTrace.GetLogger(nameof(NUnitFrameworkDriver)); + private readonly Version _nunitVersion; + #if NETFRAMEWORK private readonly NUnitFrameworkApi _api; @@ -35,8 +39,9 @@ public NUnitFrameworkDriver(AppDomain testDomain, string id, AssemblyName nunitR Guard.ArgumentNotNull(nunitRef); ID = id; + _nunitVersion = nunitRef.Version.ShouldNotBeNull(); - if (nunitRef.Version >= MINIMUM_NUNIT_VERSION) + if (_nunitVersion >= MINIMUM_NUNIT_VERSION_FOR_2018_API) { API = "2018"; _api = (NUnitFrameworkApi)testDomain.CreateInstanceFromAndUnwrap( @@ -69,6 +74,8 @@ internal NUnitFrameworkDriver(AppDomain testDomain, string api, string id, Assem Guard.ArgumentNotNullOrEmpty(id); Guard.ArgumentNotNull(nunitRef); + _nunitVersion = nunitRef.Version.ShouldNotBeNull(); + ID = id; API = api; @@ -99,6 +106,7 @@ public NUnitFrameworkDriver(string id, AssemblyName nunitRef) ID = id; API = "2018"; + _nunitVersion = nunitRef.Version.ShouldNotBeNull(); _api = new NUnitFrameworkApi2018(ID, nunitRef); } @@ -152,8 +160,19 @@ public void RunAsync(ITestEventListener? listener, string filter) => /// /// Cancel the ongoing test run. If no test is running, the call is ignored. /// - /// If true, cancel any ongoing test threads, otherwise wait for them to complete. - public void StopRun(bool force) => _api.StopRun(force); + public void RequestStop() => _api.RequestStop(); + + /// + /// Force the current test run to stop, killing threads or processes if necessary. + /// If no tests are running, the call is ignored. + /// + public void ForcedStop() + { + if (_api.ForcedStopSupported) + _api.ForcedStop(); + else + Environment.Exit(AgentExitCodes.CANCELLED_BY_FORCED_STOP); + } /// /// Returns information about the tests in an assembly. diff --git a/src/NUnitCommon/nunit.agent.core/Runners/TestAgentRunner.cs b/src/NUnitCommon/nunit.agent.core/Runners/TestAgentRunner.cs index 4c8a3819d..149415949 100644 --- a/src/NUnitCommon/nunit.agent.core/Runners/TestAgentRunner.cs +++ b/src/NUnitCommon/nunit.agent.core/Runners/TestAgentRunner.cs @@ -244,25 +244,13 @@ public AsyncTestEngineResult RunAsync(ITestEventListener? listener, TestFilter f /// Request the current test run to stop. If no tests are running, /// the call is ignored. /// - public void RequestStop() => StopRun(false); + public void RequestStop() => GetLoadedDriver().RequestStop(); /// /// Force the current test run to stop, killing threads or processes if necessary. /// If no tests are running, the call is ignored. /// - public void ForcedStop() => StopRun(true); - - private void StopRun(bool force) - { - try - { - GetLoadedDriver().StopRun(force); - } - catch (Exception ex) when (!(ex is NUnitEngineException)) - { - throw new NUnitEngineException("An exception occurred in the driver while stopping the run.", ex); - } - } + public void ForcedStop() => GetLoadedDriver().ForcedStop(); private IFrameworkDriver GetLoadedDriver() { diff --git a/src/NUnitCommon/nunit.common/AgentExitCodes.cs b/src/NUnitCommon/nunit.common/AgentExitCodes.cs index 17b381be9..df3b8b92a 100644 --- a/src/NUnitCommon/nunit.common/AgentExitCodes.cs +++ b/src/NUnitCommon/nunit.common/AgentExitCodes.cs @@ -10,6 +10,7 @@ public static class AgentExitCodes public const int DEBUGGER_SECURITY_VIOLATION = -3; public const int DEBUGGER_NOT_IMPLEMENTED = -4; public const int UNABLE_TO_LOCATE_AGENCY = -5; + public const int CANCELLED_BY_FORCED_STOP = -6; public const int UNEXPECTED_EXCEPTION = -100; public const int STACK_OVERFLOW_EXCEPTION = -1073741571; } diff --git a/src/NUnitEngine/nunit.engine.tests/Runners/MasterTestRunnerTests.cs b/src/NUnitEngine/nunit.engine.tests/Runners/MasterTestRunnerTests.cs index c8111141a..0fd42eb2a 100644 --- a/src/NUnitEngine/nunit.engine.tests/Runners/MasterTestRunnerTests.cs +++ b/src/NUnitEngine/nunit.engine.tests/Runners/MasterTestRunnerTests.cs @@ -94,16 +94,13 @@ public void RunAsync() _engineRunner.Received().Run(Arg.Any(), filter); } - [TestCase(false)] - [TestCase(true)] - public void StopRun(bool force) + [Test] + public void RequestStop() { _masterTestRunner.GetEngineRunner(); - _masterTestRunner.StopRun(force); - if (force) - _engineRunner.Received().ForcedStop(); - else - _engineRunner.Received().RequestStop(); + _masterTestRunner.RequestStop(); + + _engineRunner.Received().RequestStop(); } #endif } diff --git a/src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs b/src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs index 41e8edccf..523205b9f 100644 --- a/src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs +++ b/src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs @@ -1,5 +1,7 @@ // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt +#define USE_WORK_ITEM_TRACKER + using System; using System.ComponentModel; using System.Diagnostics; @@ -51,7 +53,9 @@ public class MasterTestRunner : ITestRunner private bool _disposed; private TestEventDispatcher _eventDispatcher = new TestEventDispatcher(); +#if USE_WORK_ITEM_TRACKER private WorkItemTracker _workItemTracker = new WorkItemTracker(); +#endif private int _testRunTimeout; private Timer? _testRunTimer; @@ -186,19 +190,16 @@ public ITestRun RunAsync(ITestEventListener? listener, TestFilter filter) /// /// Cancel the ongoing test run. If no test is running, the call is ignored. /// - /// If true, cancel any ongoing test threads, otherwise wait for them to complete. - public void StopRun(bool force) - { - if (_engineRunner is null) - return; // No test is was even started. + public void RequestStop() => _engineRunner?.RequestStop(); - if (!force) - _engineRunner.RequestStop(); - else + public void ForcedStop() + { + if (_engineRunner is not null) { _engineRunner.ForcedStop(); - // Frameworks should handle StopRun(true) by cancelling all tests and notifying +#if USE_WORK_ITEM_TRACKER + // Framework drivers should handle ForcedStop() by cancelling all tests and notifying // us of the completion of any tests that were running. However, this feature // may be absent in some frameworks or may be broken and we may not pass on the // notifications needed by some runners. In fact, such a bug is present in the @@ -225,6 +226,7 @@ public void StopRun(bool force) _engineRunner.Unload(); } +#endif } } @@ -454,9 +456,11 @@ private int CountTests(TestFilter filter) /// A TestEngineResult giving the result of the test execution private TestEngineResult RunTests(ITestEventListener? listener, TestFilter filter) { - _workItemTracker.Clear(); _eventDispatcher.Listeners.Clear(); +#if USE_WORK_ITEM_TRACKER + _workItemTracker.Clear(); _eventDispatcher.Listeners.Add(_workItemTracker); +#endif if (listener is not null) _eventDispatcher.Listeners.Add(listener); @@ -530,15 +534,18 @@ private TestEngineResult RunTests(ITestEventListener? listener, TestFilter filte private void OnTestRunTimeout(object? sender, ElapsedEventArgs e) { // Unlikely as it is, let's see if we can stop this cooperatively - StopRun(false); + RequestStop(); + // TODO: Should we wait for run complete if we are not using WorkItemTracker? // We wait for the cooperative stop and do a forced stop if it fails. // WaitForCompletion will actually be called twice, once here and - // again in StopRun(true), so the wait is doubled but no harm done. - // StopRun(True) also calls _workItemTracker to try to fix up any + // again in ForcedStop(), so the wait is doubled but no harm done. + // ForcedStop() also calls _workItemTracker to try to fix up any // in-flight items and produce as informative a result as possible. +#if USE_WORK_ITEM_TRACKER if (!_workItemTracker.WaitForCompletion(WAIT_FOR_CANCEL_TO_COMPLETE)) - StopRun(true); + ForcedStop(); +#endif } private AsyncTestEngineResult RunTestsAsync(ITestEventListener? listener, TestFilter filter)