diff --git a/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs b/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs index 29730f382..d8180862a 100644 --- a/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs +++ b/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs @@ -24,6 +24,7 @@ public class InstrumentationRunner private const string ShortMessageVariableName = "shortMsg"; private const string ProcessCrashedShortMessage = "Process crashed"; private const string InstrumentationResultPrefix = "INSTRUMENTATION_RESULT:"; + private const string TestResultsFileType = "test-results"; private readonly ILogger _logger; private readonly AdbRunner _runner; @@ -105,6 +106,13 @@ public ExitCode RunApkInstrumentation( // Determine exit code and emit summary after all operations complete ExitCode exitCode = DetermineExitCode(result, logCatSucceeded, processCrashed, failurePullingFiles, instrumentationExitCode, expectedExitCode); + + // Some applications report a zero exit code even when tests failed so we double check the test results + if (exitCode == ExitCode.SUCCESS && expectedExitCode == (int)ExitCode.SUCCESS && ContainsFailedTests(producedFiles)) + { + exitCode = ExitCode.TESTS_FAILED; + } + EmitRunSummary(exitCode, instrumentationExitCode, producedFiles, outputDirectory); return exitCode; @@ -148,6 +156,30 @@ private ExitCode DetermineExitCode(ProcessExecutionResults result, bool logCatSu return ExitCode.SUCCESS; } + private bool ContainsFailedTests(List producedFiles) + { + bool failedTestsFound = false; + + foreach (var resultFile in producedFiles.Where(file => file.Type == TestResultsFileType && !string.IsNullOrEmpty(file.Path))) + { + int? failedTests = TestResultsAnalyzer.GetFailedTestCount(resultFile.Path!); + + if (failedTests is null) + { + _logger.LogDebug($"Unable to determine the number of failed tests from '{resultFile.Path}'"); + continue; + } + + if (failedTests > 0) + { + _logger.LogError($"Instrumentation reported a successful exit code but '{resultFile.Name}' contains {failedTests} failed test(s)"); + failedTestsFound = true; + } + } + + return failedTestsFound; + } + private (int? ExitCode, bool Crashed, bool FilePullFailed) ParseInstrumentationResult(string apkPackageName, string outputDirectory, string result, List producedFiles) { // This is where test instrumentation can communicate outwardly that test execution failed @@ -231,7 +263,7 @@ private bool PullResultXMLs(string apkPackageName, string outputDirectory, IRead producedFiles.Add(new DiagnosticsFile { Name = Path.GetFileName(resultFile), - Type = "test-results", + Type = TestResultsFileType, Path = Path.Combine(outputDirectory, Path.GetFileName(resultFile)), }); } diff --git a/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs new file mode 100644 index 000000000..46fb86e3c --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.XHarness.Android; + +/// +/// Reads test result files (XML) produced by test applications and tells whether they contain failed tests. +/// This is needed because some applications report a zero instrumentation exit code even when tests failed. +/// +public static class TestResultsAnalyzer +{ + /// + /// Returns the number of failed tests found in given test results file. + /// Returns null when the file is missing or its format is not recognized (in which case we can't tell). + /// + public static int? GetFailedTestCount(string resultsFilePath) + { + if (string.IsNullOrEmpty(resultsFilePath) || !File.Exists(resultsFilePath)) + { + return null; + } + + XDocument document; + try + { + document = XDocument.Load(resultsFilePath); + } + catch (Exception) + { + return null; + } + + XElement? root = document.Root; + if (root == null) + { + return null; + } + + switch (root.Name.LocalName) + { + // xUnit v2 format (the default of the XHarness test runners) + case "assemblies": + case "assembly": + var assemblies = root.Name.LocalName == "assembly" + ? new[] { root } + : root.Elements().Where(e => e.Name.LocalName == "assembly").ToArray(); + + // An empty element is a valid result file with no failures + return assemblies.Sum(assembly => GetIntAttribute(assembly, "failed") + GetIntAttribute(assembly, "errors")); + + // NUnit v2 format + case "test-results": + return GetIntAttribute(root, "failures") + GetIntAttribute(root, "errors"); + + // NUnit v3 format + case "test-run": + return GetIntAttribute(root, "failed"); + + default: + return null; + } + } + + private static int GetIntAttribute(XElement element, string attributeName) + => int.TryParse(element.Attribute(attributeName)?.Value, out int value) ? value : 0; +} diff --git a/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs new file mode 100644 index 000000000..154796bff --- /dev/null +++ b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.IO; +using Xunit; + +namespace Microsoft.DotNet.XHarness.Android.Tests; + +public class TestResultsAnalyzerTests : IDisposable +{ + private readonly string _tempDirectory; + + public TestResultsAnalyzerTests() + { + _tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(_tempDirectory); + } + + public void Dispose() + { + if (Directory.Exists(_tempDirectory)) + { + Directory.Delete(_tempDirectory, true); + } + } + + [Fact] + public void XUnitResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @" + + "); + + Assert.Equal(3, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void XUnitResultsWithoutFailuresAreDetected() + { + var path = WriteResultsFile( + @" + + "); + + Assert.Equal(0, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void XUnitErrorsAreCountedAsFailures() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(1, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void NUnitV2ResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(3, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void NUnitV3ResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(2, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void EmptyXUnitResultsMeanNoFailures() + { + var path = WriteResultsFile(@""); + + Assert.Equal(0, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void UnknownFormatIsNotEvaluated() + { + var path = WriteResultsFile(@""); + + Assert.Null(TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void MalformedFileIsNotEvaluated() + { + var path = WriteResultsFile("this is not XML"); + + Assert.Null(TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void MissingFileIsNotEvaluated() + => Assert.Null(TestResultsAnalyzer.GetFailedTestCount(Path.Combine(_tempDirectory, "does-not-exist.xml"))); + + private string WriteResultsFile(string content) + { + var path = Path.Combine(_tempDirectory, "testResults.xml"); + File.WriteAllText(path, content); + return path; + } +} diff --git a/tests/integration-tests/Android/Commands.Tests.proj b/tests/integration-tests/Android/Commands.Tests.proj index ecc42db1f..105237250 100644 --- a/tests/integration-tests/Android/Commands.Tests.proj +++ b/tests/integration-tests/Android/Commands.Tests.proj @@ -6,21 +6,20 @@ System.Numerics.Vectors.Tests - $(AssetsBaseUri)/android/test-apk/x86/$(TestPackageName)-x86.zip - $(ArtifactsTmpDir)test-app\android\x86 + x86_64 + $(AssetsBaseUri)/android/test-apk/$(TestArchitecture)/$(TestPackageName)-$(TestArchitecture).apk + $(ArtifactsTmpDir)test-app\android\$(TestArchitecture) - + - - - + net.dot.$(TestPackageName) net.dot.MonoRunner 00:30:00 @@ -28,10 +27,10 @@