From eb039ba7b08d062beacc61e208ba6ea21f67698a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:08:09 +0000 Subject: [PATCH 1/4] Initial plan From 9c54ed8ec3d06bddc08cfd342bec21df78f51a88 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:18:30 +0000 Subject: [PATCH 2/4] Fail Android runs when pulled test results contain failed tests Co-authored-by: vitek-karas <10670590+vitek-karas@users.noreply.github.com> --- .../InstrumentationRunner.cs | 34 +++++- .../TestResultsAnalyzer.cs | 76 +++++++++++++ .../TestResultsAnalyzerTests.cs | 106 ++++++++++++++++++ 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs create mode 100644 tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs diff --git a/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs b/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs index 29730f382..d8180862a 100644 --- a/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs +++ b/src/Microsoft.DotNet.XHarness.Android/InstrumentationRunner.cs @@ -24,6 +24,7 @@ public class InstrumentationRunner private const string ShortMessageVariableName = "shortMsg"; private const string ProcessCrashedShortMessage = "Process crashed"; private const string InstrumentationResultPrefix = "INSTRUMENTATION_RESULT:"; + private const string TestResultsFileType = "test-results"; private readonly ILogger _logger; private readonly AdbRunner _runner; @@ -105,6 +106,13 @@ public ExitCode RunApkInstrumentation( // Determine exit code and emit summary after all operations complete ExitCode exitCode = DetermineExitCode(result, logCatSucceeded, processCrashed, failurePullingFiles, instrumentationExitCode, expectedExitCode); + + // Some applications report a zero exit code even when tests failed so we double check the test results + if (exitCode == ExitCode.SUCCESS && expectedExitCode == (int)ExitCode.SUCCESS && ContainsFailedTests(producedFiles)) + { + exitCode = ExitCode.TESTS_FAILED; + } + EmitRunSummary(exitCode, instrumentationExitCode, producedFiles, outputDirectory); return exitCode; @@ -148,6 +156,30 @@ private ExitCode DetermineExitCode(ProcessExecutionResults result, bool logCatSu return ExitCode.SUCCESS; } + private bool ContainsFailedTests(List producedFiles) + { + bool failedTestsFound = false; + + foreach (var resultFile in producedFiles.Where(file => file.Type == TestResultsFileType && !string.IsNullOrEmpty(file.Path))) + { + int? failedTests = TestResultsAnalyzer.GetFailedTestCount(resultFile.Path!); + + if (failedTests is null) + { + _logger.LogDebug($"Unable to determine the number of failed tests from '{resultFile.Path}'"); + continue; + } + + if (failedTests > 0) + { + _logger.LogError($"Instrumentation reported a successful exit code but '{resultFile.Name}' contains {failedTests} failed test(s)"); + failedTestsFound = true; + } + } + + return failedTestsFound; + } + private (int? ExitCode, bool Crashed, bool FilePullFailed) ParseInstrumentationResult(string apkPackageName, string outputDirectory, string result, List producedFiles) { // This is where test instrumentation can communicate outwardly that test execution failed @@ -231,7 +263,7 @@ private bool PullResultXMLs(string apkPackageName, string outputDirectory, IRead producedFiles.Add(new DiagnosticsFile { Name = Path.GetFileName(resultFile), - Type = "test-results", + Type = TestResultsFileType, Path = Path.Combine(outputDirectory, Path.GetFileName(resultFile)), }); } diff --git a/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs new file mode 100644 index 000000000..c8a0eea25 --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.XHarness.Android; + +/// +/// Reads test result files (XML) produced by test applications and tells whether they contain failed tests. +/// This is needed because some applications report a zero instrumentation exit code even when tests failed. +/// +public static class TestResultsAnalyzer +{ + /// + /// Returns the number of failed tests found in given test results file. + /// Returns null when the file is missing or its format is not recognized (in which case we can't tell). + /// + public static int? GetFailedTestCount(string resultsFilePath) + { + if (string.IsNullOrEmpty(resultsFilePath) || !File.Exists(resultsFilePath)) + { + return null; + } + + XDocument document; + try + { + document = XDocument.Load(resultsFilePath); + } + catch (Exception) + { + return null; + } + + XElement? root = document.Root; + if (root == null) + { + return null; + } + + switch (root.Name.LocalName) + { + // xUnit v2 format (the default of the XHarness test runners) + case "assemblies": + case "assembly": + var assemblies = root.Name.LocalName == "assembly" + ? new[] { root } + : root.Elements().Where(e => e.Name.LocalName == "assembly").ToArray(); + + if (assemblies.Length == 0) + { + return null; + } + + return assemblies.Sum(assembly => GetIntAttribute(assembly, "failed") + GetIntAttribute(assembly, "errors")); + + // NUnit v2 format + case "test-results": + return GetIntAttribute(root, "failures") + GetIntAttribute(root, "errors"); + + // NUnit v3 format + case "test-run": + return GetIntAttribute(root, "failed"); + + default: + return null; + } + } + + private static int GetIntAttribute(XElement element, string attributeName) + => int.TryParse(element.Attribute(attributeName)?.Value, out int value) ? value : 0; +} diff --git a/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs new file mode 100644 index 000000000..78ef4dae1 --- /dev/null +++ b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs @@ -0,0 +1,106 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.IO; +using Xunit; + +namespace Microsoft.DotNet.XHarness.Android.Tests; + +public class TestResultsAnalyzerTests : IDisposable +{ + private readonly string _tempDirectory; + + public TestResultsAnalyzerTests() + { + _tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(_tempDirectory); + } + + public void Dispose() + { + if (Directory.Exists(_tempDirectory)) + { + Directory.Delete(_tempDirectory, true); + } + } + + [Fact] + public void XUnitResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @" + + "); + + Assert.Equal(3, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void XUnitResultsWithoutFailuresAreDetected() + { + var path = WriteResultsFile( + @" + + "); + + Assert.Equal(0, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void XUnitErrorsAreCountedAsFailures() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(1, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void NUnitV2ResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(3, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void NUnitV3ResultsWithFailuresAreDetected() + { + var path = WriteResultsFile( + @""); + + Assert.Equal(2, TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void UnknownFormatIsNotEvaluated() + { + var path = WriteResultsFile(@""); + + Assert.Null(TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void MalformedFileIsNotEvaluated() + { + var path = WriteResultsFile("this is not XML"); + + Assert.Null(TestResultsAnalyzer.GetFailedTestCount(path)); + } + + [Fact] + public void MissingFileIsNotEvaluated() + => Assert.Null(TestResultsAnalyzer.GetFailedTestCount(Path.Combine(_tempDirectory, "does-not-exist.xml"))); + + private string WriteResultsFile(string content) + { + var path = Path.Combine(_tempDirectory, "testResults.xml"); + File.WriteAllText(path, content); + return path; + } +} From d1db8593f8b0fd1891b70de75b2965a64ff49100 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:23:26 +0000 Subject: [PATCH 3/4] Treat empty xUnit assemblies element as zero failures; add test Co-authored-by: vitek-karas <10670590+vitek-karas@users.noreply.github.com> --- .../TestResultsAnalyzer.cs | 6 +----- .../TestResultsAnalyzerTests.cs | 8 ++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs index c8a0eea25..46fb86e3c 100644 --- a/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs +++ b/src/Microsoft.DotNet.XHarness.Android/TestResultsAnalyzer.cs @@ -51,11 +51,7 @@ public static class TestResultsAnalyzer ? new[] { root } : root.Elements().Where(e => e.Name.LocalName == "assembly").ToArray(); - if (assemblies.Length == 0) - { - return null; - } - + // An empty element is a valid result file with no failures return assemblies.Sum(assembly => GetIntAttribute(assembly, "failed") + GetIntAttribute(assembly, "errors")); // NUnit v2 format diff --git a/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs index 78ef4dae1..154796bff 100644 --- a/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs +++ b/tests/Microsoft.DotNet.XHarness.Android.Tests/TestResultsAnalyzerTests.cs @@ -77,6 +77,14 @@ public void NUnitV3ResultsWithFailuresAreDetected() Assert.Equal(2, TestResultsAnalyzer.GetFailedTestCount(path)); } + [Fact] + public void EmptyXUnitResultsMeanNoFailures() + { + var path = WriteResultsFile(@""); + + Assert.Equal(0, TestResultsAnalyzer.GetFailedTestCount(path)); + } + [Fact] public void UnknownFormatIsNotEvaluated() { From 73457bfc65f57bae23012032d90ee5cb521d0a7f Mon Sep 17 00:00:00 2001 From: vitek-karas <10670590+vitek-karas@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:47:26 +0200 Subject: [PATCH 4/4] Use updated x86_64 Android test APK Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 628dfcba-913b-421e-b2c8-202a7459abd6 --- .../integration-tests/Android/Commands.Tests.proj | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/integration-tests/Android/Commands.Tests.proj b/tests/integration-tests/Android/Commands.Tests.proj index ecc42db1f..105237250 100644 --- a/tests/integration-tests/Android/Commands.Tests.proj +++ b/tests/integration-tests/Android/Commands.Tests.proj @@ -6,21 +6,20 @@ System.Numerics.Vectors.Tests - $(AssetsBaseUri)/android/test-apk/x86/$(TestPackageName)-x86.zip - $(ArtifactsTmpDir)test-app\android\x86 + x86_64 + $(AssetsBaseUri)/android/test-apk/$(TestArchitecture)/$(TestPackageName)-$(TestArchitecture).apk + $(ArtifactsTmpDir)test-app\android\$(TestArchitecture) - + - - - + net.dot.$(TestPackageName) net.dot.MonoRunner 00:30:00 @@ -28,10 +27,10 @@