From eedad80fe94a1920eeb65735de8f4c6bd8d42ea4 Mon Sep 17 00:00:00 2001 From: carzh Date: Wed, 15 Jan 2025 12:33:56 -0800 Subject: [PATCH 1/4] Working android MAUI app + project that runs android browserstack test --- .../.config/dotnet-tools.json | 13 +++ .../BrowserStackTest.cs | 74 ++++++++++++ ...rosoft.ML.OnnxRuntime.Tests.Android.csproj | 22 ++++ .../RunAllTest.cs | 107 ++++++++++++++++++ .../browserstack.yml | 13 +++ ...Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj | 11 +- 6 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json new file mode 100644 index 0000000000000..67d39c423d4d7 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "browserstack-sdk": { + "version": "1.16.13", + "commands": [ + "browserstack-sdk" + ], + "rollForward": false + } + } +} \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs new file mode 100644 index 0000000000000..594ba4f0e5cf9 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs @@ -0,0 +1,74 @@ +using Newtonsoft.Json; +using NUnit.Framework.Interfaces; +using NUnit.Framework; +using OpenQA.Selenium; +using OpenQA.Selenium.Appium; +using OpenQA.Selenium.Appium.Android; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.OnnxRuntime.Tests.Android +{ + public class BrowserStackTest + { + public AndroidDriver driver; + public BrowserStackTest() { } + + [SetUp] + public void Init() + { + var androidOptions = new AppiumOptions + { + AutomationName = "UIAutomator2", + PlatformName = "Android", + }; + + driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions); + } + + /// + /// Sends a log to BrowserStack that is visible in the text logs and labelled as ANNOTATION. + /// + /// Log text to send. + /// Log level -- choose between info, debug, warning, and error + public void browserStackLog(String text, String logLevel = "info") + { + String jsonToSend = String.Format("browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", JsonConvert.ToString(text), JsonConvert.ToString(logLevel)); + + ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend); + } + + /// + /// Passes the correct test status to BrowserStack and ensures the driver quits. + /// + [TearDown] + public void Dispose() + { + try + { + // According to https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail + // BrowserStack doesn't know whether test assertions have passed or failed. Below handles + // passing the test status to BrowserStack along with any relevant information. + if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed) + { + String failureMessage = TestContext.CurrentContext.Result.Message; + String jsonToSendFailure = String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"failed\", \"reason\": {0}}}", JsonConvert.ToString(failureMessage)); + + ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure); + } + else + { + ((IJavaScriptExecutor)driver).ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"passed\", \"reason\": \"\"}}"); + } + } + finally + { + // will run even if exception is thrown by previous block + ((AndroidDriver)driver).Quit(); + } + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj new file mode 100644 index 0000000000000..9b9028d30cc5b --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs new file mode 100644 index 0000000000000..72eb720a891ca --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs @@ -0,0 +1,107 @@ +using OpenQA.Selenium.Appium; +using OpenQA.Selenium; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.OnnxRuntime.Tests.Android +{ + [TestFixture] + public class RunAllTest : BrowserStackTest + { + public AppiumElement FindAppiumElement(String xpathQuery, String text) + { + IReadOnlyCollection appiumElements = driver.FindElements(By.XPath(xpathQuery)); + + foreach (var element in appiumElements) + { + if (element.Text.Contains(text)) + { + return element; + } + } + // was unable to find given element + throw new Exception(String.Format("Could not find {0}: {1} on the page.", xpathQuery, text)); + } + + public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text) + { + AppiumElement appiumElement = FindAppiumElement(xpathQuery, text); + appiumElement.Click(); + return appiumElement; + } + + public (int, int) GetPassFailCount() + { + int numPassed = -1; + int numFailed = -1; + + IReadOnlyCollection labelElements = driver.FindElements(By.XPath("//android.widget.TextView")); + + for (int i = 0; i < labelElements.Count; i++) + { + AppiumElement element = labelElements.ElementAt(i); + + if (element.Text.Equals("✔")) + { + i++; + numPassed = int.Parse(labelElements.ElementAt(i).Text); + } + + if (element.Text.Equals("⛔")) + { + i++; + numFailed = int.Parse(labelElements.ElementAt(i).Text); + break; + } + } + + Assert.That(numPassed, Is.GreaterThanOrEqualTo(0), "Could not find number passed label."); + Assert.That(numFailed, Is.GreaterThanOrEqualTo(0), "Could not find number failed label."); + + return (numPassed, numFailed); + } + + [Test] + public async Task ClickRunAllTest() + { + AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All"); + + while (!runAllButton.Enabled) + { + // waiting for unit tests to execute + await Task.Delay(500); + } + + var (numPassed, numFailed) = GetPassFailCount(); + + if (numFailed == 0) + { + Assert.Pass(); + return; + } + + // click into test results if tests have failed + FindAppiumElementThenClick("//android.widget.TextView", "⛔"); + await Task.Delay(500); + FindAppiumElementThenClick("//android.widget.EditText", "All"); + await Task.Delay(100); + FindAppiumElementThenClick("//android.widget.TextView", "Failed"); + await Task.Delay(500); + + StringBuilder sb = new StringBuilder(); + sb.AppendLine("PASSED TESTS: " + numPassed + " | FAILED TESTS: " + numFailed); + + IReadOnlyCollection textResults = driver.FindElements(By.XPath("//android.widget.TextView")); + foreach (var element in textResults) + { + sb.AppendLine(element.Text); + } + + Assert.That(numFailed, Is.EqualTo(0), sb.ToString()); + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml new file mode 100644 index 0000000000000..9efbc9fc6aa61 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml @@ -0,0 +1,13 @@ +app: ..\Microsoft.ML.OnnxRuntime.Tests.MAUI\bin\Release\net8.0-android\publish\ORT.CSharp.Tests.MAUI-Signed.apk +platforms: + - platformName: android + deviceName: Samsung Galaxy S22 Ultra + platformVersion: 12.0 +browserstackLocal: true +buildName: ORT android test +buildIdentifier: ${BUILD_NUMBER} +projectName: ORT-UITests +debug: true +networkLogs: false +testContextOptions: + skipSessionStatus: true \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj index e07448daeea7f..c4a547748ac8c 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj @@ -7,7 +7,7 @@ - net8.0-android;net8.0-ios;net8.0-maccatalyst + net8.0;net8.0-android;net8.0-ios;net8.0-maccatalyst $(TargetFrameworks);net8.0-windows10.0.19041.0 - Exe + Exe Microsoft.ML.OnnxRuntime.Tests.MAUI true true @@ -101,6 +101,7 @@ + @@ -122,4 +123,10 @@ + + + false + en + + From 45db126a6bca9ff26b7eb10188f085da3bda78c7 Mon Sep 17 00:00:00 2001 From: carzh Date: Thu, 16 Jan 2025 14:20:21 -0800 Subject: [PATCH 2/4] wrote readme, added device runner comments, renamed project to BrowserStack.Android, applied suggestions --- .../.config/dotnet-tools.json | 0 .../BrowserStackTest.cs | 26 ++- ...Runtime.Tests.BrowserStack.Android.csproj} | 0 .../README.md | 48 ++++ .../RunAllTest.cs | 23 +- .../browserstack.yml | 0 ...Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj | 211 +++++++++--------- 7 files changed, 190 insertions(+), 118 deletions(-) rename csharp/test/{Microsoft.ML.OnnxRuntime.Tests.Android => Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android}/.config/dotnet-tools.json (100%) rename csharp/test/{Microsoft.ML.OnnxRuntime.Tests.Android => Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android}/BrowserStackTest.cs (64%) rename csharp/test/{Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj => Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android.csproj} (100%) create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/README.md rename csharp/test/{Microsoft.ML.OnnxRuntime.Tests.Android => Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android}/RunAllTest.cs (74%) rename csharp/test/{Microsoft.ML.OnnxRuntime.Tests.Android => Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android}/browserstack.yml (100%) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/.config/dotnet-tools.json similarity index 100% rename from csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json rename to csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/.config/dotnet-tools.json diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs similarity index 64% rename from csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs rename to csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs index 594ba4f0e5cf9..a2802cbfabfa8 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs @@ -10,18 +10,18 @@ using System.Text; using System.Threading.Tasks; -namespace Microsoft.ML.OnnxRuntime.Tests.Android +namespace Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android { public class BrowserStackTest { public AndroidDriver driver; - public BrowserStackTest() { } + public BrowserStackTest() + {} [SetUp] public void Init() { - var androidOptions = new AppiumOptions - { + var androidOptions = new AppiumOptions { AutomationName = "UIAutomator2", PlatformName = "Android", }; @@ -30,13 +30,15 @@ public void Init() } /// - /// Sends a log to BrowserStack that is visible in the text logs and labelled as ANNOTATION. + /// Sends a log to BrowserStack that is visible in the text logs and labeled as ANNOTATION. /// /// Log text to send. /// Log level -- choose between info, debug, warning, and error public void browserStackLog(String text, String logLevel = "info") { - String jsonToSend = String.Format("browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", JsonConvert.ToString(text), JsonConvert.ToString(logLevel)); + String jsonToSend = String.Format( + "browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", + JsonConvert.ToString(text), JsonConvert.ToString(logLevel)); ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend); } @@ -49,19 +51,25 @@ public void Dispose() { try { - // According to https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail + // According to + // https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail // BrowserStack doesn't know whether test assertions have passed or failed. Below handles // passing the test status to BrowserStack along with any relevant information. if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed) { String failureMessage = TestContext.CurrentContext.Result.Message; - String jsonToSendFailure = String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"failed\", \"reason\": {0}}}", JsonConvert.ToString(failureMessage)); + String jsonToSendFailure = + String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": " + + "{\"status\":\"failed\", \"reason\": {0}}}", + JsonConvert.ToString(failureMessage)); ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure); } else { - ((IJavaScriptExecutor)driver).ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"passed\", \"reason\": \"\"}}"); + ((IJavaScriptExecutor)driver) + .ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": " + + "{\"status\":\"passed\", \"reason\": \"\"}}"); } } finally diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android.csproj similarity index 100% rename from csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj rename to csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android.csproj diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/README.md b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/README.md new file mode 100644 index 0000000000000..9c4e2307d8f70 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/README.md @@ -0,0 +1,48 @@ +# BrowserStack Android test +This project will run the Android MAUI tests on BrowserStack, which allows you to run automated tests on a variety of mobile devices. + +## Context +Microsoft.ML.OnnxRuntime.Tests.MAUI uses DeviceRunners.VisualRunners to allow running the unit tests (found in Microsoft.ML.OnnxRuntime.Tests.Common) across multiple devices. DeviceRunners.VisualRunners provides a simple UI with a button that will run the unit tests and a panel with the unit test results. + +In order to automate the process of running the unit tests across mobile devices, Appium is used for UI testing orchestration (it provides a way to interact with the UI), and BrowserStack automatically runs these Appium tests across different mobile devices. + +This project does not include the capability to start an Appium server locally or attach to a local emulator or device. + +## Build & run instructions +### Requirements +* A BrowserStack account with access to App Automate + * You can set BrowserStack credentials as environment variables as shown [here](https://www.browserstack.com/docs/app-automate/appium/getting-started/c-sharp/nunit/integrate-your-tests#CLI) +* ONNXRuntime NuGet package + 1. You can either download the [stable NuGet package](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime) then follow the instructions from [NativeLibraryInclude.props file](../Microsoft.ML.OnnxRuntime.Tests.Common/NativeLibraryInclude.props) to use the downloaded .nupkg file + 2. Or follow the [build instructions](https://onnxruntime.ai/docs/build/android.html) to build the Android package locally +* The dotnet workloads for maui and maui-android, which will not always automatically install correctly + 1. `dotnet workload install maui` + 2. `dotnet workload install maui-android` +* [Appium](https://appium.io/docs/en/latest/quickstart/) and the [UiAutomator2 driver](https://appium.io/docs/en/latest/quickstart/uiauto2-driver/) + +### Run instructions +1. Build the Microsoft.ML.OnnxRuntime.Tests.MAUI project into a signed APK. + 1. Run the following: `dotnet publish -c Release -f net8.0-android` in the Microsoft.ML.OnnxRuntime.Tests.MAUI directory. + 2. Search for the APK files generated. They should be located in `bin\Release\net8.0-android\publish`. + 3. If they're in a different location, edit the `browserstack.yml` file to target the path to the signed APK. +2. Ensure you've set the BrowserStack credentials as environment variables. +3. Run the following in the Microsoft.ML.OnnxRuntime.Tests.Android.BrowserStack directory: `dotnet test` +4. Navigate to the [BrowserStack App Automate dashboard](https://app-automate.browserstack.com/dashboard/v2/builds) to see your test running! + +## Troubleshooting & Resources +### BrowserStack Resources +- [Configuration docs](https://www.browserstack.com/docs/app-automate/appium/sdk-params#test-context) for browserstack.yml +- [Configuration generator](https://www.browserstack.com/docs/app-automate/capabilities) for browserstack.yml +- [Integration guide](https://www.browserstack.com/docs/app-automate/appium/getting-started/c-sharp/nunit/integrate-your-tests#CLI) + +### Troubleshooting +- Issues building the MAUI app: + - Make sure that the maui and maui-android workloads are installed correctly by running `dotnet workload list` + - If you believe the issues are workload related, you can also try running `dotnet workload repair` (this has personally never worked for me) + - Try running `dotnet clean`. However, this does not fully remove all the previous intermediaries. If you're still running into the errors, manually deleting the bin and obj folders can sometimes resolve them. +- After building the MAUI app, try installing on an emulator and clicking the "Run All" button to ensure that everything is working. (If you are missing the ONNXRuntime package, it will not show up as an error until you click "Run All".) + - Running the MAUI app from Visual Studio will not replicate running it through BrowserStack. Instead, use `adb install [path to signed apk]` to install the app then use the emulator to launch the app. +- Issues with the Android.BrowserStack test app: there is an Appium Doctor package on npm -- run `npm install @appium/doctor --location=global` then `appium-doctor --android` and follow the directed instructions. Some errors with Appium on Android will not appear until runtime. +- Connection refused by Appium server: this can happen if you already have an Appium server running locally. If you do, stop the Appium server then try `dotnet test` again. +- App is crashing on BrowserStack or it emits an error that it cannot run this APK file: make sure that you are passing in the correct signed APK from the publish folder. +- It appears that a test runs on CLI but a build is not launched on BrowserStack: this happens when the BrowserStack Test Adapter cannot find the browserstack.yml file (which has to be named "browserstack.yml" -- do not be tricked by BrowserStack's article on custom-named configuration files) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs similarity index 74% rename from csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs rename to csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs index 72eb720a891ca..e4a546fdda888 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs @@ -7,8 +7,18 @@ using System.Text; using System.Threading.Tasks; -namespace Microsoft.ML.OnnxRuntime.Tests.Android +namespace Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android { + /// + /// This class contains a single test: RunAll, which interacts with the UI from + /// https://github.com/mattleibow/DeviceRunners/tree/main by clicking the "Run All" button and checking the number + /// of passed and failed tests. + /// + /// It searches for elements on the page using Appium's WebDriver. These searches use the XPath attributes. + /// + /// Launching the MAUI test app in Appium Inspector will allow you to see the exact XPath attributes for each + /// element. + /// [TestFixture] public class RunAllTest : BrowserStackTest { @@ -39,7 +49,8 @@ public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text) int numPassed = -1; int numFailed = -1; - IReadOnlyCollection labelElements = driver.FindElements(By.XPath("//android.widget.TextView")); + IReadOnlyCollection labelElements = + driver.FindElements(By.XPath("//android.widget.TextView")); for (int i = 0; i < labelElements.Count; i++) { @@ -68,8 +79,10 @@ public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text) [Test] public async Task ClickRunAllTest() { + // XAML for the main page: + // https://github.com/mattleibow/DeviceRunners/blob/main/src/DeviceRunners.VisualRunners.Maui/App/Pages/HomePage.xaml AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All"); - + while (!runAllButton.Enabled) { // waiting for unit tests to execute @@ -87,6 +100,10 @@ public async Task ClickRunAllTest() // click into test results if tests have failed FindAppiumElementThenClick("//android.widget.TextView", "⛔"); await Task.Delay(500); + + // Brings you to the test assembly page + // XAML for test assembly page: + // https://github.com/mattleibow/DeviceRunners/blob/cba7644e07b305ba64dc930b01c3eee55ef2b93d/src/DeviceRunners.VisualRunners.Maui/App/Pages/TestAssemblyPage.xaml FindAppiumElementThenClick("//android.widget.EditText", "All"); await Task.Delay(100); FindAppiumElementThenClick("//android.widget.TextView", "Failed"); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/browserstack.yml similarity index 100% rename from csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml rename to csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/browserstack.yml diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj index c4a547748ac8c..652da8899f6a9 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj @@ -1,128 +1,127 @@  - - $(ProjectDir)..\..\.. - + + $(ProjectDir)..\..\.. + - + - - - net8.0;net8.0-android;net8.0-ios;net8.0-maccatalyst - $(TargetFrameworks);net8.0-windows10.0.19041.0 + + + net8.0-android;net8.0-ios;net8.0-maccatalyst + $(TargetFrameworks);net8.0-windows10.0.19041.0 - - - - Exe - Microsoft.ML.OnnxRuntime.Tests.MAUI - true - true - enable - enable - true - - 8002 - - - $(DefineConstants);INCLUDE_FAILING_TESTS - $(DefineConstants);MODE_NON_INTERACTIVE_VISUAL - $(DefineConstants);MODE_XHARNESS - - - Microsoft.ML.OnnxRuntime.Tests.MAUI - - - ORT.CSharp.Tests.MAUI - - - 1.0 - 1 - - 15.0 - 13.1 - 30.0 - 10.0.17763.0 - 10.0.17763.0 - - true - ..\..\OnnxRuntime.snk - - - - - - - - - - - - - - - - - - - - - + + Microsoft.ML.OnnxRuntime.Tests.MAUI + true + true + enable + enable + true + + 8002 + + + $(DefineConstants);INCLUDE_FAILING_TESTS + $(DefineConstants);MODE_NON_INTERACTIVE_VISUAL + $(DefineConstants);MODE_XHARNESS + + + Microsoft.ML.OnnxRuntime.Tests.MAUI + + + ORT.CSharp.Tests.MAUI + + + 1.0 + 1 + + 15.0 + 13.1 + 30.0 + 10.0.17763.0 + 10.0.17763.0 + + true + ..\..\OnnxRuntime.snk + + + + + + + + + + + + + + + + + + + + + - - - InferenceTest.cs - - - OrtIoBindingAllocationTest.cs - - - TensorTests.cs - - - - - + + InferenceTest.cs + + + OrtIoBindingAllocationTest.cs + + + TensorTests.cs + + + + + - - - + - - - - - + + + + + - - - - - - - - - - - - - - - - <_VisualStudioTestRunnerFiles + + + + + + + + + + + + + + + + <_VisualStudioTestRunnerFiles Include="@(PackagingOutputs)" Condition="$([System.String]::Copy('%(PackagingOutputs.FullPath)').Contains('xunit.runner.visualstudio'))" /> - - - + + + false From 123b10e4708270c4dd6d4fa784f80d1637a4facf Mon Sep 17 00:00:00 2001 From: carzh Date: Fri, 17 Jan 2025 15:53:15 -0800 Subject: [PATCH 3/4] switched gh xaml link to permalink + removed Assert.Pass --- .../RunAllTest.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs index e4a546fdda888..5db3dc9957d1c 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/RunAllTest.cs @@ -80,7 +80,7 @@ public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text) public async Task ClickRunAllTest() { // XAML for the main page: - // https://github.com/mattleibow/DeviceRunners/blob/main/src/DeviceRunners.VisualRunners.Maui/App/Pages/HomePage.xaml + // https://github.com/mattleibow/DeviceRunners/blob/cba7644e07b305ba64dc930b01c3eee55ef2b93d/src/DeviceRunners.VisualRunners.Maui/App/Pages/HomePage.xaml AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All"); while (!runAllButton.Enabled) @@ -93,7 +93,6 @@ public async Task ClickRunAllTest() if (numFailed == 0) { - Assert.Pass(); return; } From fbee995e4edb62ff2bc484713c20ae44eeae9501 Mon Sep 17 00:00:00 2001 From: carzh Date: Tue, 21 Jan 2025 11:23:33 -0800 Subject: [PATCH 4/4] removed the browserstack log func --- .../BrowserStackTest.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs index a2802cbfabfa8..84377d65d1213 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android/BrowserStackTest.cs @@ -29,20 +29,6 @@ public void Init() driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions); } - /// - /// Sends a log to BrowserStack that is visible in the text logs and labeled as ANNOTATION. - /// - /// Log text to send. - /// Log level -- choose between info, debug, warning, and error - public void browserStackLog(String text, String logLevel = "info") - { - String jsonToSend = String.Format( - "browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", - JsonConvert.ToString(text), JsonConvert.ToString(logLevel)); - - ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend); - } - /// /// Passes the correct test status to BrowserStack and ensures the driver quits. ///