diff --git a/tests/MSTest.Android.Tests/Libraries/JavaInterop/InteropTests.cs b/tests/MSTest.Android.Tests/Libraries/JavaInterop/InteropTests.cs new file mode 100644 index 00000000000..f8fff1acfe4 --- /dev/null +++ b/tests/MSTest.Android.Tests/Libraries/JavaInterop/InteropTests.cs @@ -0,0 +1,50 @@ +using AndroidMSTestProof.TestLibrary; +using Java.Interop; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: DoNotParallelize] + +namespace AndroidMSTestProof.JavaInterop; + +[TestClass] +public class InteropTests : LifecycleBase +{ + [TestMethod] + [TestCategory ("proof")] + [TestCategory ("jni")] + public void ReferencedJNI () + { + AssertInitialized (); + using var value = new Java.Lang.String ("referenced JNI"); + Assert.AreEqual ("referenced JNI", value.ToString ()); + Assert.AreNotEqual (IntPtr.Zero, JniEnvironment.EnvironmentPointer); + Context.WriteLine ("[D ASSEMBLY] " + typeof (InteropTests).Assembly.GetName ().Name); + } + +#pragma warning disable MSTEST0087 // Matching display names deliberately test distinct JNI row identities. + [TestMethod] + [TestCategory ("proof")] + [TestCategory ("jni")] + [DataRow ("first", DisplayName = "same JNI display")] + [DataRow ("second", DisplayName = "same JNI display")] + public void StringRoundtrip (string text) + { + AssertInitialized (); + var reference = JniEnvironment.Strings.NewString (text); + try { + Assert.IsTrue (reference.IsValid); + Assert.AreEqual (text, JniEnvironment.Strings.ToString (reference)); + } finally { + JniObjectReference.Dispose (ref reference); + } + } +#pragma warning restore MSTEST0087 + + [TestMethod] + [TestCategory ("diagnostic")] + public void GeneratedInteropFailure () + { + AssertInitialized (); + Assert.Fail ("intentional referenced-JavaInterop generated invocation failure"); + } +} diff --git a/tests/MSTest.Android.Tests/Libraries/JavaInterop/MSTest.Android.JavaInterop.Tests.csproj b/tests/MSTest.Android.Tests/Libraries/JavaInterop/MSTest.Android.JavaInterop.Tests.csproj new file mode 100644 index 00000000000..a688dfdb38a --- /dev/null +++ b/tests/MSTest.Android.Tests/Libraries/JavaInterop/MSTest.Android.JavaInterop.Tests.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/MSTest.Android.Tests/Libraries/TestLibrary/LifecycleTests.cs b/tests/MSTest.Android.Tests/Libraries/TestLibrary/LifecycleTests.cs new file mode 100644 index 00000000000..a1d69acafe9 --- /dev/null +++ b/tests/MSTest.Android.Tests/Libraries/TestLibrary/LifecycleTests.cs @@ -0,0 +1,82 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: DoNotParallelize] + +namespace AndroidMSTestProof.TestLibrary; + +public abstract class LifecycleBase +{ + int initialized; + int cleaned; + + public TestContext? TestContext { get; set; } + + protected TestContext Context => TestContext ?? throw new InvalidOperationException ("MSTest did not assign the inherited context."); + + [TestInitialize] + public void Initialize () + { + Assert.AreEqual (1, ++initialized); + Assert.AreEqual (0, cleaned); + using var value = new Java.Lang.String ("initialize"); + Assert.AreEqual ("initialize", value.ToString ()); + Context.WriteLine ($"[D INITIALIZE] {GetType ().FullName} {Context.TestName} calls={initialized}"); + } + + [TestCleanup] + public void Cleanup () + { + Assert.AreEqual (1, initialized); + Assert.AreEqual (1, ++cleaned); + using var value = new Java.Lang.String ("cleanup"); + Assert.AreEqual ("cleanup", value.ToString ()); + Context.WriteLine ($"[D CLEANUP] {GetType ().FullName} {Context.TestName} calls={cleaned} outcome={Context.CurrentTestOutcome}"); + } + + protected void AssertInitialized () + { + Assert.AreEqual (1, initialized); + Assert.AreEqual (0, cleaned); + } +} + +[TestClass] +public class LibraryTests : LifecycleBase +{ + [TestMethod] + [TestCategory ("proof")] + [TestCategory ("library")] + public void ReferencedLibrary () + { + AssertInitialized (); + Assert.AreEqual (typeof (LibraryTests).Assembly, GetType ().Assembly); + Context.WriteLine ("[D ASSEMBLY] " + typeof (LibraryTests).Assembly.GetName ().Name); + } + + [TestMethod] + [TestCategory ("proof")] + [TestCategory ("library")] + public async Task AsyncLibrary () + { + AssertInitialized (); + await Task.Delay (1); + AssertInitialized (); + } + + [TestMethod] + [TestCategory ("proof")] + [TestCategory ("library")] + public void InconclusiveLibrary () + { + AssertInitialized (); + Assert.Inconclusive ("intentional D inconclusive; inherited cleanup must execute"); + } + + [TestMethod] + [TestCategory ("diagnostic")] + public void GeneratedLibraryFailure () + { + AssertInitialized (); + Assert.Fail ("intentional referenced-library generated invocation failure"); + } +} diff --git a/tests/MSTest.Android.Tests/Libraries/TestLibrary/MSTest.Android.TestLibrary.Tests.csproj b/tests/MSTest.Android.Tests/Libraries/TestLibrary/MSTest.Android.TestLibrary.Tests.csproj new file mode 100644 index 00000000000..1cc587947b5 --- /dev/null +++ b/tests/MSTest.Android.Tests/Libraries/TestLibrary/MSTest.Android.TestLibrary.Tests.csproj @@ -0,0 +1,3 @@ + + + diff --git a/tests/MSTest.Android.Tests/MSTest.Android.Tests.csproj b/tests/MSTest.Android.Tests/MSTest.Android.Tests.csproj new file mode 100644 index 00000000000..3bc7c7ea5e6 --- /dev/null +++ b/tests/MSTest.Android.Tests/MSTest.Android.Tests.csproj @@ -0,0 +1,19 @@ + + + + Exe + net.dot.android.mstestproof + net.dot.android.mstestproof.Instrumentation + true + true + true + + + + + + + + + + diff --git a/tests/MSTest.Android.Tests/ProofTests.cs b/tests/MSTest.Android.Tests/ProofTests.cs new file mode 100644 index 00000000000..dd1f5bb7fb7 --- /dev/null +++ b/tests/MSTest.Android.Tests/ProofTests.cs @@ -0,0 +1,106 @@ +using System.Reflection; +using AndroidMSTestProof.TestLibrary; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: DoNotParallelize] + +namespace AndroidMSTestProof; + +[TestClass] +public class ProofTests : LifecycleBase +{ + [TestMethod] + [TestCategory ("proof")] + public void AndroidBinding () + { + AssertInitialized (); + Assert.IsTrue (Android.OS.Build.VERSION.SdkInt >= Android.OS.BuildVersionCodes.N); + Assert.AreEqual ("Mono.Android", typeof (Java.Lang.String).Assembly.GetName ().Name); + } + + [TestMethod] + [TestCategory ("proof")] + public async Task AsyncTask () + { + AssertInitialized (); + await Task.Delay (1); + AssertInitialized (); + using var value = new Java.Lang.String ("async JNI"); + Assert.AreEqual ("async JNI", value.ToString ()); + } + + [TestMethod] + [TestCategory ("proof")] + public async ValueTask AsyncValueTask () + { + AssertInitialized (); + await Task.Delay (1); + AssertInitialized (); + } + +#pragma warning disable MSTEST0042, MSTEST0087 // Identical arguments and display names deliberately test row identity. + [TestMethod] + [TestCategory ("proof")] + [DataRow (0, DisplayName = "same data display")] + [DataRow (0, DisplayName = "same data display")] + [DataRow (1, DisplayName = "same data display")] + public void DataRows (int row) + { + AssertInitialized (); + Assert.IsTrue (row is 0 or 1); + Context.WriteLine ($"[D ROW] DataRows value={row}"); + } +#pragma warning restore MSTEST0042, MSTEST0087 + + public static IEnumerable DynamicCases { + get { + yield return [0]; + yield return [0]; + yield return [1]; + } + } + + public static string DynamicDisplay (MethodInfo method, object [] data) => "same dynamic display"; + + [TestMethod] + [TestCategory ("proof")] + [DynamicData (nameof (DynamicCases), DynamicDataDisplayName = nameof (DynamicDisplay))] + public async Task DynamicRows (int row) + { + AssertInitialized (); + await Task.Delay (1); + Assert.IsTrue (row is 0 or 1); + Context.WriteLine ($"[D ROW] DynamicRows value={row}"); + } + + [TestMethod] + [TestCategory ("proof")] + public void Attachment () + { + AssertInitialized (); + var directory = Android.App.Application.Context.GetExternalFilesDir (null)?.AbsolutePath + ?? throw new InvalidOperationException ("No Android external results directory."); + var path = Path.Combine (directory, "TestResults", "d-proof-attachment.txt"); + File.WriteAllText (path, "D attachment\nline two"); + Context.AddResultFile (path); + Context.WriteLine ("[D ATTACHMENT] " + path); + } + + [TestMethod] + [TestCategory ("proof")] + [Ignore ("intentional D ignored test; no initialization or cleanup expected")] + public void Ignored () + { + Assert.Fail ("an ignored test must never execute"); + } + + [TestMethod] + [TestCategory ("diagnostic")] + public void GeneratedFailure () + { + AssertInitialized (); +#pragma warning disable MSTEST0025 // Preserve an actual comparison failure and its generated invocation stack. + Assert.AreEqual ("expected D value", "actual D value", "intentional diagnostic failure\nsecond line"); +#pragma warning restore MSTEST0025 + } +} diff --git a/tests/MSTest.Android.Tests/README.md b/tests/MSTest.Android.Tests/README.md new file mode 100644 index 00000000000..1f01a18cbb5 --- /dev/null +++ b/tests/MSTest.Android.Tests/README.md @@ -0,0 +1,135 @@ +# Native Android MSTest source-generation proof + +This isolated application tests the native MSTest/Microsoft.Testing.Platform +integration. It does not migrate `Mono.Android.NET-Tests`, Java.Interop's NUnit +suite, or JCW tests. Its scoped package pins are MSTest and SourceGeneration +`4.6.0-preview.26524.1` and MTP `2.6.0-preview.26524.1`; product pins are unchanged. + +The app registers its own assembly and both referenced test-library assemblies +together through `AddMSTest`. Each project references the generator directly. +Runtime packages explicitly specify `Publish="true"` because the SDK otherwise +excludes `PrivateAssets="all"` packages from Android's publish/deployment closure. +The generator remains analyzer-only and private. + +## Cases and evidence + +| Selection | Rows | Expected outcome | +| --- | ---: | --- | +| `proof` (default) | 17 | 15 passed, 2 skipped | +| `diagnostic` | 3 | 3 deliberate failures, native exit 2 | +| `proof,diagnostic` | 20 | 15 passed, 2 skipped, 3 deliberate failures | +| `jni` | 3 | 3 passed | +| `library` | 3 | 2 passed, 1 inconclusive/skip | + +Coverage includes inherited `TestContext` assignment and initialization/cleanup +in all three assemblies, ignored versus inconclusive lifecycle, Task and +ValueTask, duplicate DataRow arguments, DynamicData, matching display names, +Android bindings, raw JNI string-reference disposal, real TRX attachments and +multiline failure diagnostics with generated invocation stacks. + +Native UIDs, not display names, identify results. `D.Nodes.xml` preserves every +native update, its state and native method metadata. On this Android runtime the +adapter's native assembly-full-name field is empty; the separate defining +assembly comes from the explicitly registered fixture `Type`, with namespace +and type-name matching and rejection of unknown/mismatched metadata. + +Execution includes native discovery through MSTest's source runner and records +every resulting native UID. Standalone `--list-tests json` was a separate +experiment: CoreCLR emitted the expected identities, but NativeAOT did not yield +capturable JSON. That surface is not claimed as proven, and `-e list` is rejected. +Neither console redirection nor reflection observers remain in this app. + +The instrumentation publishes the shared Android streaming protocol and native +TRX path. It recreates only this app's external `TestResults` directory per run. +Result files, including MSTest's initially owner-only attachments, are explicitly +made readable for ADB export. These files contain synthetic proof evidence. +Attachment creation uses Android's explicit external directory: bundled +assemblies have an empty `Assembly.Location`, so MSTest's default +source-derived `TestContext.TestResultsDirectory` is not a portable writable +Android directory. This proof does not replace that framework behavior or +establish a general NUnit synchronization-context compatibility layer. + +## Running + +Prepare/build the local SDK normally, then use an explicitly owned device and +ADB server. Do not borrow another session's writable toolchain, caches or AVD. +For example, with `ANDROID_SERIAL` and `ADB_SERVER_SOCKET` already configured: + +```sh +./dotnet-local.sh build tests/MSTest.Android.Tests/MSTest.Android.Tests.csproj \ + -t:Install -c Release "-p:AdbTarget=-s $ANDROID_SERIAL" \ + -p:PublishTrimmed=true -p:TrimMode=full \ + -p:EmitCompilerGeneratedFiles=true \ + -p:CompilerGeneratedFilesOutputPath=obj/d-generated/Release + +adb -s "$ANDROID_SERIAL" logcat -c +adb -s "$ANDROID_SERIAL" shell am instrument -w -r \ + -e include proof,diagnostic \ + net.dot.android.mstestproof/net.dot.android.mstestproof.Instrumentation +adb -s "$ANDROID_SERIAL" logcat -d -v threadtime +``` + +Category selection also accepts +the `IncludeCategories` runtime configuration setting; this bounded runner +supports simple alphanumeric/hyphen category names joined by comma or semicolon. +It rejects unsupported expressions rather than silently interpreting them. + +Capture `instrumentation.txt`, `logcat.txt`, the returned counts/exit/path bundle +as `bundle.json`, the returned `D.Nodes.xml` and native `.trx`, and the complete +external `TestResults` directory as `device-results`. For execution directories +captured in that layout: + +```sh +python3 tests/MSTest.Android.Tests/verify-results.py \ + CORECLR_ROOTED CORECLR_ROOTLESS AOT_ROOTED AOT_ROOTLESS +``` + +Use `--selection proof` for default-category runs. The verifier checks the exact +method/row shape, unique and stable native UIDs, outcomes, TRX and wire identity +agreement, lifecycle multiplicity, duplicate argument values, attachment +content/linkage, generated diagnostic stacks and absence of the temporary +runtime observer. + +To exercise the actual project-mode Android MTP host (default `proof` selection), +select MTP explicitly and use an absolute project path: + +```sh +DOTNET_TEST_RUNNER=Microsoft.Testing.Platform ./dotnet-local.sh test \ + --project "$PWD/tests/MSTest.Android.Tests/MSTest.Android.Tests.csproj" \ + --no-build -c Release --report-trx \ + "-p:AdbTarget=-s $ANDROID_SERIAL" +``` + +Pass the same runtime/build properties to both build and test. In particular, +add `-p:PublishAot=true -p:Optimize=true -p:PublishTrimmed=true -p:TrimMode=full` +to both commands for NativeAOT. Both project-mode CoreCLR and NativeAOT produced +15 passed and 2 skipped rows. The console's `net11.0|arm64` label describes the +host tool; the installed app targets `net12.0-android`. + +## Trimming and runtime boundary + +The app has no manual whole-test-assembly discovery roots. Compare it against an +experimental import containing `TrimmerRootAssembly` with `RootMode="All"` for +`MSTest.Android.Tests`, `MSTest.Android.JavaInterop.Tests` and +`MSTest.Android.TestLibrary.Tests`. Run both with full trimming; then repeat +with `-p:PublishAot=true -p:Optimize=true`. Verify the produced APK contains the +NativeAOT application library, not CoreCLR, rather than relying on flags alone. + +At the selected integrated head, observer-free full-trim CoreCLR and actual +NativeAOT on API 35 arm64 produced the same 20 UID/metadata/outcome rows with +and without those three experimental roots. Both native ABIs (arm64 and x86_64) +compiled; device execution covered arm64 only. The .NET 12 SDK has CoreCLR and +NativeAOT runtime imports, not a Mono runtime import. + +Separate observed runs verified active `SourceGeneratedReflectionOperations`, +generated constructors, test/lifecycle invokers and inherited context setters +from all three assemblies. Generated descriptor completeness remained **false**. +The temporary observer is not part of this application. Generated dynamic +dependencies and bounded fallback reflection remain; compiler output and these +results do not prove universally reflection-free discovery or eliminate +framework trimming/AOT warnings. + +This result supersedes only those experimental fixture-discovery roots. +Existing NUnit whole-assembly roots, RD.xml/complete metadata, converter and +reflection roots, StartupHook, JNI and resource roots elsewhere remain +independent and untouched until their actual consumers are removed and tested. diff --git a/tests/MSTest.Android.Tests/Shared.props b/tests/MSTest.Android.Tests/Shared.props new file mode 100644 index 00000000000..7e5656d4f58 --- /dev/null +++ b/tests/MSTest.Android.Tests/Shared.props @@ -0,0 +1,18 @@ + + + + $(DotNetAndroidTargetFramework) + $(AndroidMinimumDotNetApiLevel) + enable + enable + false + false + 4.6.0-preview.26524.1 + 2.6.0-preview.26524.1 + + + + + + + diff --git a/tests/MSTest.Android.Tests/TestInstrumentation.cs b/tests/MSTest.Android.Tests/TestInstrumentation.cs new file mode 100644 index 00000000000..621a4db3d47 --- /dev/null +++ b/tests/MSTest.Android.Tests/TestInstrumentation.cs @@ -0,0 +1,205 @@ +using System.Reflection; +using System.Text; +using System.Xml.Linq; +using Android.App; +using Android.OS; +using Android.Runtime; +using Android.Util; +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Extensions.TrxReport.Abstractions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; + +namespace AndroidMSTestProof; + +[Instrumentation (Name = "net.dot.android.mstestproof.Instrumentation")] +public class TestInstrumentation : Instrumentation +{ + const string LogTag = "D.MSTest"; + Bundle? arguments; + + protected TestInstrumentation (IntPtr handle, JniHandleOwnership ownership) : base (handle, ownership) { } + + public override void OnCreate (Bundle? arguments) + { + this.arguments = arguments; + AppDomain.CurrentDomain.UnhandledException += (_, e) => Log.Error (LogTag, $"Unhandled exception: {e.ExceptionObject}"); + TaskScheduler.UnobservedTaskException += (_, e) => Log.Error (LogTag, $"Unobserved exception: {e.Exception}"); + base.OnCreate (arguments); + Start (); + } + + public override void OnStart () + { + base.OnStart (); + _ = RunAsync (); + } + + async Task RunAsync () + { + var bundle = new Bundle (); + try { + var directory = Application.Context.GetExternalFilesDir (null)?.AbsolutePath + ?? throw new InvalidOperationException ("Android external results directory is unavailable."); + var results = Path.Combine (directory, "TestResults"); + if (Directory.Exists (results)) + Directory.Delete (results, recursive: true); + Directory.CreateDirectory (results); + + Type [] fixtures = [ + typeof (ProofTests), + typeof (JavaInterop.InteropTests), + typeof (TestLibrary.LibraryTests), + ]; + Assembly [] assemblies = fixtures.Select (fixture => fixture.Assembly).Distinct ().ToArray (); + foreach (var assembly in assemblies) + Log.Info (LogTag, "[D REGISTER] " + assembly.GetName ().Name); + + var commandLine = new List { + "--results-directory", results, "--report-trx", "--progress", "off", "--output", "normal", + }; + if (arguments?.ContainsKey ("list") == true) + throw new NotSupportedException ("This representative runner records native IDs during execution; standalone listing is not part of its proven contract."); + var include = arguments?.GetString ("include") ?? AppContext.GetData ("IncludeCategories") as string ?? "proof"; + var categories = include.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (categories.Length == 0) + throw new InvalidOperationException ("An explicit nonempty proof category selection is required."); + if (categories.Any (category => category.Any (character => !char.IsAsciiLetterOrDigit (character) && character != '-'))) + throw new ArgumentException ("This bounded proof accepts only simple alphanumeric category names."); + commandLine.Add ("--filter"); + commandLine.Add ("(" + string.Join ("|", categories.Select (category => "TestCategory=" + category)) + ")"); + var consumer = new ResultConsumer (this, results, fixtures); + var builder = await TestApplication.CreateBuilderAsync (commandLine.ToArray (), new TestApplicationOptions { + EnableTelemetry = false, + }); + builder.AddMSTest (() => assemblies); + builder.AddTrxReportProvider (); + builder.TestHost.AddDataConsumer (_ => consumer); + using ITestApplication application = await builder.BuildAsync (); + int exit = await application.RunAsync (); + bundle.PutString ("nodesPath", consumer.SaveNodes ()); + foreach (var file in Directory.EnumerateFiles (results, "*", SearchOption.AllDirectories)) { + using var exported = new Java.IO.File (file); + if (!exported.SetReadable (true, ownerOnly: false)) + throw new IOException ($"Cannot export proof result file: {file}"); + } + if (consumer.Passed + consumer.Failed + consumer.Skipped == 0) + throw new InvalidOperationException ($"No results from native MSTest (exit {exit})."); + if (exit != 0 && (exit != 2 || consumer.Failed == 0)) + throw new InvalidOperationException ($"Native MSTest infrastructure failed with exit {exit}."); + if (consumer.TrxPath is not string trx || !File.Exists (trx)) + throw new InvalidOperationException ("Native MSTest did not produce its TRX."); + bundle.PutString (InstrumentationProtocol.KeyResultsPath, trx); + bundle.PutInt (InstrumentationProtocol.KeyPassedCount, consumer.Passed); + bundle.PutInt (InstrumentationProtocol.KeyFailedCount, consumer.Failed); + bundle.PutInt (InstrumentationProtocol.KeySkippedCount, consumer.Skipped); + bundle.PutInt ("nativeExit", exit); + Finish (Result.Ok, bundle); + } catch (Exception ex) { + Log.Error (LogTag, ex.ToString ()); + bundle.PutString (InstrumentationProtocol.KeyError, ex.ToString ()); + Finish (Result.Canceled, bundle); + } + } + + sealed class ResultConsumer (Instrumentation instrumentation, string resultsDirectory, Type [] fixtures) : IDataConsumer + { + readonly List nodes = []; + readonly Dictionary fixtureTypes = fixtures.ToDictionary (fixture => + fixture.FullName ?? throw new InvalidOperationException ("A proof fixture has no full name.")); + int passed, failed, skipped; + public int Passed => passed; + public int Failed => failed; + public int Skipped => skipped; + public string? TrxPath { get; private set; } + public string Uid => nameof (ResultConsumer); + public string DisplayName => nameof (ResultConsumer); + public string Description => "D native Android result and artifact evidence"; + public string Version => "1.0"; + public Task IsEnabledAsync () => Task.FromResult (true); + public Type [] DataTypesConsumed => [typeof (TestNodeUpdateMessage), typeof (SessionFileArtifact)]; + + public string SaveNodes () + { + var path = Path.Combine (resultsDirectory, "D.Nodes.xml"); + lock (nodes) + new XDocument (new XElement ("nodes", nodes)).Save (path); + return path; + } + + public Task ConsumeAsync (IDataProducer producer, IData data, CancellationToken cancellationToken) + { + if (data is SessionFileArtifact artifact) { + Log.Info (LogTag, "[D ARTIFACT] " + artifact.FileInfo.FullName); + if (artifact.FileInfo.Extension == ".trx") + TrxPath = artifact.FileInfo.FullName; + return Task.CompletedTask; + } + if (data is not TestNodeUpdateMessage { TestNode: var node }) + throw new InvalidOperationException ($"Unexpected native message: {data.GetType ().FullName}"); + + var state = node.Properties.SingleOrDefault (); + var method = node.Properties.SingleOrDefault (); + string? outcome = state switch { + PassedTestNodeStateProperty => InstrumentationProtocol.OutcomePassed, + FailedTestNodeStateProperty or ErrorTestNodeStateProperty or TimeoutTestNodeStateProperty => InstrumentationProtocol.OutcomeFailed, + SkippedTestNodeStateProperty => InstrumentationProtocol.OutcomeSkipped, + _ => null, + }; + string qualified = method is null ? "" : $"{method.Namespace}.{method.TypeName}.{method.MethodName}"; + string definingAssembly = ""; + if (method is not null) { + string typeName = $"{method.Namespace}.{method.TypeName}"; + if (!fixtureTypes.TryGetValue (typeName, out var fixture)) + throw new InvalidOperationException ($"Native metadata names an unregistered proof fixture: {typeName}"); + definingAssembly = fixture.Assembly.FullName ?? throw new InvalidOperationException ("A proof assembly has no full name."); + if (!string.IsNullOrEmpty (method.AssemblyFullName) && method.AssemblyFullName != definingAssembly) + throw new InvalidOperationException ($"Native assembly metadata does not match the registered fixture: {typeName}"); + } + lock (nodes) { + nodes.Add (new XElement ("node", + new XAttribute ("uid", node.Uid.Value), + new XAttribute ("assembly", definingAssembly), + new XAttribute ("nativeAssembly", method?.AssemblyFullName ?? ""), + new XAttribute ("namespace", method?.Namespace ?? ""), + new XAttribute ("type", method?.TypeName ?? ""), + new XAttribute ("method", method?.MethodName ?? ""), + new XAttribute ("display", node.DisplayName), + new XAttribute ("state", state?.GetType ().Name ?? ""), + node.Properties.OfType ().Select (trait => + new XElement ("trait", new XAttribute ("key", trait.Key), new XAttribute ("value", trait.Value))))); + } + Log.Info (LogTag, $"[D NODE] uid={node.Uid.Value} method={qualified} display={node.DisplayName} state={state?.GetType ().Name}"); + if (outcome is null && state is not InProgressTestNodeStateProperty) + return Task.CompletedTask; + if (outcome == InstrumentationProtocol.OutcomePassed) + Interlocked.Increment (ref passed); + else if (outcome == InstrumentationProtocol.OutcomeFailed) + Interlocked.Increment (ref failed); + else if (outcome == InstrumentationProtocol.OutcomeSkipped) + Interlocked.Increment (ref skipped); + var status = new Bundle (); + status.PutString (InstrumentationProtocol.KeyEvent, outcome is null ? InstrumentationProtocol.EventStart : InstrumentationProtocol.EventFinish); + status.PutString (InstrumentationProtocol.KeyTest, node.Uid.Value); + status.PutString (InstrumentationProtocol.KeyName, node.DisplayName); + status.PutString (InstrumentationProtocol.KeyClass, method is null ? "" : $"{method.Namespace}.{method.TypeName}"); + if (outcome is not null) + status.PutString (InstrumentationProtocol.KeyOutcome, outcome); + var error = node.Properties.SingleOrDefault (); + string? message = error?.Message ?? state switch { + FailedTestNodeStateProperty failure => failure.Exception?.Message ?? failure.Explanation, + ErrorTestNodeStateProperty failure => failure.Exception?.Message ?? failure.Explanation, + TimeoutTestNodeStateProperty failure => failure.Exception?.Message ?? failure.Explanation, + SkippedTestNodeStateProperty skip => skip.Explanation, + _ => null, + }; + if (message is not null) + status.PutString (InstrumentationProtocol.KeyMessageBase64, Convert.ToBase64String (Encoding.UTF8.GetBytes (message))); + if (error?.StackTrace is string stack) + status.PutString (InstrumentationProtocol.KeyStackBase64, Convert.ToBase64String (Encoding.UTF8.GetBytes (stack))); + instrumentation.SendStatus (0, status); + return Task.CompletedTask; + } + } +} diff --git a/tests/MSTest.Android.Tests/verify-results.py b/tests/MSTest.Android.Tests/verify-results.py new file mode 100644 index 00000000000..8b78be0c82b --- /dev/null +++ b/tests/MSTest.Android.Tests/verify-results.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Verify captured native results, including identities, lifecycle and artifacts.""" + +import argparse +import base64 +from collections import Counter +import json +from pathlib import Path +import xml.etree.ElementTree as ET + +NS = {"t": "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"} +FIXTURES = { + "AndroidMSTestProof.ProofTests": ( + "MSTest.Android.Tests", + {"AndroidBinding": 1, "AsyncTask": 1, "AsyncValueTask": 1, "DataRows": 3, + "DynamicRows": 3, "Attachment": 1, "Ignored": 1, "GeneratedFailure": 1}), + "AndroidMSTestProof.TestLibrary.LibraryTests": ( + "MSTest.Android.TestLibrary.Tests", + {"ReferencedLibrary": 1, "AsyncLibrary": 1, "InconclusiveLibrary": 1, "GeneratedLibraryFailure": 1}), + "AndroidMSTestProof.JavaInterop.InteropTests": ( + "MSTest.Android.JavaInterop.Tests", + {"ReferencedJNI": 1, "StringRoundtrip": 2, "GeneratedInteropFailure": 1}), +} +STATES = { + "PassedTestNodeStateProperty": "passed", + "FailedTestNodeStateProperty": "failed", + "SkippedTestNodeStateProperty": "skipped", +} +FAILURES = {"GeneratedFailure", "GeneratedLibraryFailure", "GeneratedInteropFailure"} +SKIPS = {"Ignored", "InconclusiveLibrary"} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def verify(directory, selection): + bundle = json.loads((directory / "bundle.json").read_text()) + expected = Counter() + for classname, (assembly, methods) in FIXTURES.items(): + for method, count in methods.items(): + if selection == "proof" and method in FAILURES: + continue + if selection == "diagnostic" and method not in FAILURES: + continue + expected[assembly, classname, method] = count + finals = [node for node in ET.parse(directory / "D.Nodes.xml").getroot() if node.get("state") in STATES] + actual = Counter((node.get("assembly", "").split(",")[0], f'{node.get("namespace")}.{node.get("type")}', node.get("method")) for node in finals) + require(actual == expected, f"{directory}: missing {expected - actual}; extra {actual - expected}") + by_uid = {node.get("uid"): node for node in finals} + require(len(by_uid) == len(finals), f"{directory}: duplicate native UIDs") + for node in finals: + wanted = "failed" if node.get("method") in FAILURES else "skipped" if node.get("method") in SKIPS else "passed" + require(STATES[node.get("state")] == wanted, f"{directory}: unexpected outcome {node.attrib}") + outcomes = Counter(STATES[node.get("state")] for node in finals) + for outcome in ("passed", "failed", "skipped"): + require(int(bundle[outcome]) == outcomes[outcome], f"{directory}: inconsistent {outcome} bundle") + require(int(bundle["nativeExit"]) == (2 if outcomes["failed"] else 0), f"{directory}: wrong native exit") + reports = list(directory.glob("*.trx")) + require(len(reports) == 1, f"{directory}: expected one native TRX") + results = ET.parse(reports[0]).findall(".//t:Results/t:UnitTestResult", NS) + require(Counter(result.get("testId") for result in results) == Counter(by_uid.keys()), f"{directory}: native TRX UID mismatch") + row_values = {"DataRows": Counter(), "DynamicRows": Counter()} + for result in results: + node = by_uid[result.get("testId")] + outcome = STATES[node.get("state")] + require(result.get("outcome") == {"passed": "Passed", "failed": "Failed", "skipped": "NotExecuted"}[outcome], + f"{directory}: native TRX outcome mismatch") + stdout = result.findtext("t:Output/t:StdOut", "", NS) + method = node.get("method") + if method == "Ignored": + require("[D INITIALIZE]" not in stdout and "[D CLEANUP]" not in stdout, f"{directory}: ignored lifecycle ran") + else: + require(stdout.count("[D INITIALIZE]") == stdout.count("[D CLEANUP]") == 1, + f"{directory}: missing or repeated lifecycle for {node.attrib}") + require("calls=1" in stdout, f"{directory}: incorrect lifecycle count") + if method in row_values: + lines = [line for line in stdout.splitlines() if line.startswith(f"[D ROW] {method} value=")] + require(len(lines) == 1, f"{directory}: row argument evidence missing") + row_values[method][int(lines[0].partition("value=")[2])] += 1 + if outcome == "failed": + stack = result.findtext("t:Output/t:ErrorInfo/t:StackTrace", "", NS) + require("MSTest.SourceGenerated.MSTestReflectionMetadata" in stack, f"{directory}: no generated invocation stack") + if selection != "diagnostic": + for method, values in row_values.items(): + require(values == {0: 2, 1: 1}, f"{directory}: lost argument multiplicity for {method}") + attachments = list((directory / "device-results").rglob("d-proof-attachment.txt")) + require(attachments, f"{directory}: no exported real attachment") + require(all(path.read_text() == "D attachment\nline two" for path in attachments), f"{directory}: attachment content mismatch") + require(any(result.findall(".//t:ResultFile", NS) for result in results), f"{directory}: attachment not linked from TRX") + events, current = [], {} + for line in (directory / "instrumentation.txt").read_text().splitlines(): + if line.startswith("INSTRUMENTATION_STATUS: "): + key, separator, value = line.removeprefix("INSTRUMENTATION_STATUS: ").partition("=") + require(bool(separator), f"{directory}: invalid protocol field") + current[key] = value + elif line.startswith("INSTRUMENTATION_STATUS_CODE: "): + events.append(current) + current = {} + finishes = [event for event in events if event.get("event") == "finish"] + require(Counter((event["test"], event["outcome"]) for event in finishes) == + Counter((uid, STATES[node.get("state")]) for uid, node in by_uid.items()), f"{directory}: protocol UID/outcome mismatch") + for event in finishes: + if event["outcome"] == "failed": + require("intentional" in base64.b64decode(event["message-b64"]).decode(), f"{directory}: failure message not exported") + require("MSTest.SourceGenerated.MSTestReflectionMetadata" in base64.b64decode(event["stack-b64"]).decode(), + f"{directory}: generated stack not exported") + require(not list((directory / "device-results").rglob("D.Sourcegen.xml")), f"{directory}: temporary observer is still present") + return Counter((node.get("uid"), node.get("assembly"), node.get("namespace"), node.get("type"), + node.get("method"), node.get("display"), STATES[node.get("state")]) for node in finals) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directories", nargs="+", type=Path) + parser.add_argument("--selection", choices=("all", "proof", "diagnostic"), default="all") + args = parser.parse_args() + reference = None + for directory in args.directories: + actual = verify(directory, args.selection) + if reference is not None: + require(actual == reference, f"{directory}: native identity/multiplicity/outcome changed") + reference = actual + print(f"{directory}: {sum(actual.values())} exact native rows verified") + + +if __name__ == "__main__": + main()