Skip to content

Commit 2fc75a4

Browse files
authored
[mobile] Add Android BrowserStack test project back (#23551)
## Description Follow-up for #23383 and #23474 * Adds android BrowserStack test back in * Modifies MAUI csproj file to build into an APK ### Motivation and Context There were 2 issues with the previous PRs: 1. The updated MAUI .csproj file configuration failed when building to iOS and MacCatalyst. This caused problems in the packaging pipeline because we build all C# projects in the .soln file in the packaging pipeline. Removed the Mac & iOS build targets for now 3. The previous MAUI .csproj file configuration did not build into an APK. It was missing the `<OutputType>` XAML tag and the Android package type XAML tag.
1 parent 9e18b6a commit 2fc75a4

File tree

7 files changed

+418
-117
lines changed

7 files changed

+418
-117
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"version": 1,
3+
"isRoot": true,
4+
"tools": {
5+
"browserstack-sdk": {
6+
"version": "1.16.13",
7+
"commands": [
8+
"browserstack-sdk"
9+
],
10+
"rollForward": false
11+
}
12+
}
13+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using Newtonsoft.Json;
2+
using NUnit.Framework.Interfaces;
3+
using NUnit.Framework;
4+
using OpenQA.Selenium;
5+
using OpenQA.Selenium.Appium;
6+
using OpenQA.Selenium.Appium.Android;
7+
using System;
8+
using System.Collections.Generic;
9+
using System.Linq;
10+
using System.Text;
11+
using System.Threading.Tasks;
12+
13+
namespace Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android
14+
{
15+
public class BrowserStackTest
16+
{
17+
public AndroidDriver driver;
18+
public BrowserStackTest()
19+
{}
20+
21+
[SetUp]
22+
public void Init()
23+
{
24+
var androidOptions = new AppiumOptions {
25+
AutomationName = "UIAutomator2",
26+
PlatformName = "Android",
27+
};
28+
29+
driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions);
30+
}
31+
32+
/// <summary>
33+
/// Passes the correct test status to BrowserStack and ensures the driver quits.
34+
/// </summary>
35+
[TearDown]
36+
public void Dispose()
37+
{
38+
try
39+
{
40+
// According to
41+
// https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail
42+
// BrowserStack doesn't know whether test assertions have passed or failed. Below handles
43+
// passing the test status to BrowserStack along with any relevant information.
44+
if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed)
45+
{
46+
String failureMessage = TestContext.CurrentContext.Result.Message;
47+
String jsonToSendFailure =
48+
String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": " +
49+
"{\"status\":\"failed\", \"reason\": {0}}}",
50+
JsonConvert.ToString(failureMessage));
51+
52+
((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure);
53+
}
54+
else
55+
{
56+
((IJavaScriptExecutor)driver)
57+
.ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": " +
58+
"{\"status\":\"passed\", \"reason\": \"\"}}");
59+
}
60+
}
61+
finally
62+
{
63+
// will run even if exception is thrown by previous block
64+
((AndroidDriver)driver).Quit();
65+
}
66+
}
67+
}
68+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
8+
<IsPackable>false</IsPackable>
9+
<IsTestProject>true</IsTestProject>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<PackageReference Include="Appium.WebDriver" Version="5.0.0-rc.5" />
14+
<PackageReference Include="BrowserStack.TestAdapter" Version="0.13.13" />
15+
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
16+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
17+
<PackageReference Include="NUnit" Version="3.13.0" />
18+
<PackageReference Include="NUnit.Analyzers" Version="3.3.0" />
19+
<PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
20+
</ItemGroup>
21+
22+
</Project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# BrowserStack Android test
2+
This project will run the Android MAUI tests on BrowserStack, which allows you to run automated tests on a variety of mobile devices.
3+
4+
## Context
5+
Microsoft.ML.OnnxRuntime.Tests.MAUI uses DeviceRunners.VisualRunners to allow running the unit tests (found in Microsoft.ML.OnnxRuntime.Tests.Common) across multiple devices. DeviceRunners.VisualRunners provides a simple UI with a button that will run the unit tests and a panel with the unit test results.
6+
7+
In order to automate the process of running the unit tests across mobile devices, Appium is used for UI testing orchestration (it provides a way to interact with the UI), and BrowserStack automatically runs these Appium tests across different mobile devices.
8+
9+
This project does not include the capability to start an Appium server locally or attach to a local emulator or device.
10+
11+
## Build & run instructions
12+
### Requirements
13+
* A BrowserStack account with access to App Automate
14+
* You can set BrowserStack credentials as environment variables as shown [here](https://www.browserstack.com/docs/app-automate/appium/getting-started/c-sharp/nunit/integrate-your-tests#CLI)
15+
* ONNXRuntime NuGet package
16+
1. You can either download the [stable NuGet package](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime) then follow the instructions from [NativeLibraryInclude.props file](../Microsoft.ML.OnnxRuntime.Tests.Common/NativeLibraryInclude.props) to use the downloaded .nupkg file
17+
2. Or follow the [build instructions](https://onnxruntime.ai/docs/build/android.html) to build the Android package locally
18+
* The dotnet workloads for maui and maui-android, which will not always automatically install correctly
19+
1. `dotnet workload install maui`
20+
2. `dotnet workload install maui-android`
21+
* [Appium](https://appium.io/docs/en/latest/quickstart/) and the [UiAutomator2 driver](https://appium.io/docs/en/latest/quickstart/uiauto2-driver/)
22+
23+
### Run instructions
24+
1. Build the Microsoft.ML.OnnxRuntime.Tests.MAUI project into a signed APK.
25+
1. Run the following: `dotnet publish -c Release -f net8.0-android` in the Microsoft.ML.OnnxRuntime.Tests.MAUI directory.
26+
2. Search for the APK files generated. They should be located in `bin\Release\net8.0-android\publish`.
27+
3. If they're in a different location, edit the `browserstack.yml` file to target the path to the signed APK.
28+
2. Ensure you've set the BrowserStack credentials as environment variables.
29+
3. Run the following in the Microsoft.ML.OnnxRuntime.Tests.Android.BrowserStack directory: `dotnet test`
30+
4. Navigate to the [BrowserStack App Automate dashboard](https://app-automate.browserstack.com/dashboard/v2/builds) to see your test running!
31+
32+
## Troubleshooting & Resources
33+
### BrowserStack Resources
34+
- [Configuration docs](https://www.browserstack.com/docs/app-automate/appium/sdk-params#test-context) for browserstack.yml
35+
- [Configuration generator](https://www.browserstack.com/docs/app-automate/capabilities) for browserstack.yml
36+
- [Integration guide](https://www.browserstack.com/docs/app-automate/appium/getting-started/c-sharp/nunit/integrate-your-tests#CLI)
37+
38+
### Troubleshooting
39+
- Issues building the MAUI app:
40+
- Make sure that the maui and maui-android workloads are installed correctly by running `dotnet workload list`
41+
- If you believe the issues are workload related, you can also try running `dotnet workload repair` (this has personally never worked for me)
42+
- Try running `dotnet clean`. However, this does not fully remove all the previous intermediaries. If you're still running into the errors, manually deleting the bin and obj folders can sometimes resolve them.
43+
- After building the MAUI app, try installing on an emulator and clicking the "Run All" button to ensure that everything is working. (If you are missing the ONNXRuntime package, it will not show up as an error until you click "Run All".)
44+
- Running the MAUI app from Visual Studio will not replicate running it through BrowserStack. Instead, use `adb install [path to signed apk]` to install the app then use the emulator to launch the app.
45+
- Issues with the Android.BrowserStack test app: there is an Appium Doctor package on npm -- run `npm install @appium/doctor --location=global` then `appium-doctor --android` and follow the directed instructions. Some errors with Appium on Android will not appear until runtime.
46+
- Connection refused by Appium server: this can happen if you already have an Appium server running locally. If you do, stop the Appium server then try `dotnet test` again.
47+
- App is crashing on BrowserStack or it emits an error that it cannot run this APK file: make sure that you are passing in the correct signed APK from the publish folder.
48+
- It appears that a test runs on CLI but a build is not launched on BrowserStack: this happens when the BrowserStack Test Adapter cannot find the browserstack.yml file (which has to be named "browserstack.yml" -- do not be tricked by BrowserStack's article on custom-named configuration files)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
using OpenQA.Selenium.Appium;
2+
using OpenQA.Selenium;
3+
using NUnit.Framework;
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
10+
namespace Microsoft.ML.OnnxRuntime.Tests.BrowserStack.Android
11+
{
12+
/// <summary>
13+
/// This class contains a single test: RunAll, which interacts with the UI from
14+
/// https://github.com/mattleibow/DeviceRunners/tree/main by clicking the "Run All" button and checking the number
15+
/// of passed and failed tests.
16+
///
17+
/// It searches for elements on the page using Appium's WebDriver. These searches use the XPath attributes.
18+
///
19+
/// Launching the MAUI test app in Appium Inspector will allow you to see the exact XPath attributes for each
20+
/// element.
21+
/// </summary>
22+
[TestFixture]
23+
public class RunAllTest : BrowserStackTest
24+
{
25+
public AppiumElement FindAppiumElement(String xpathQuery, String text)
26+
{
27+
IReadOnlyCollection<AppiumElement> appiumElements = driver.FindElements(By.XPath(xpathQuery));
28+
29+
foreach (var element in appiumElements)
30+
{
31+
if (element.Text.Contains(text))
32+
{
33+
return element;
34+
}
35+
}
36+
// was unable to find given element
37+
throw new Exception(String.Format("Could not find {0}: {1} on the page.", xpathQuery, text));
38+
}
39+
40+
public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text)
41+
{
42+
AppiumElement appiumElement = FindAppiumElement(xpathQuery, text);
43+
appiumElement.Click();
44+
return appiumElement;
45+
}
46+
47+
public (int, int) GetPassFailCount()
48+
{
49+
int numPassed = -1;
50+
int numFailed = -1;
51+
52+
IReadOnlyCollection<AppiumElement> labelElements =
53+
driver.FindElements(By.XPath("//android.widget.TextView"));
54+
55+
for (int i = 0; i < labelElements.Count; i++)
56+
{
57+
AppiumElement element = labelElements.ElementAt(i);
58+
59+
if (element.Text.Equals("✔"))
60+
{
61+
i++;
62+
numPassed = int.Parse(labelElements.ElementAt(i).Text);
63+
}
64+
65+
if (element.Text.Equals("⛔"))
66+
{
67+
i++;
68+
numFailed = int.Parse(labelElements.ElementAt(i).Text);
69+
break;
70+
}
71+
}
72+
73+
Assert.That(numPassed, Is.GreaterThanOrEqualTo(0), "Could not find number passed label.");
74+
Assert.That(numFailed, Is.GreaterThanOrEqualTo(0), "Could not find number failed label.");
75+
76+
return (numPassed, numFailed);
77+
}
78+
79+
[Test]
80+
public async Task ClickRunAllTest()
81+
{
82+
// XAML for the main page:
83+
// https://github.com/mattleibow/DeviceRunners/blob/cba7644e07b305ba64dc930b01c3eee55ef2b93d/src/DeviceRunners.VisualRunners.Maui/App/Pages/HomePage.xaml
84+
AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All");
85+
86+
while (!runAllButton.Enabled)
87+
{
88+
// waiting for unit tests to execute
89+
await Task.Delay(500);
90+
}
91+
92+
var (numPassed, numFailed) = GetPassFailCount();
93+
94+
if (numFailed == 0)
95+
{
96+
return;
97+
}
98+
99+
// click into test results if tests have failed
100+
FindAppiumElementThenClick("//android.widget.TextView", "⛔");
101+
await Task.Delay(500);
102+
103+
// Brings you to the test assembly page
104+
// XAML for test assembly page:
105+
// https://github.com/mattleibow/DeviceRunners/blob/cba7644e07b305ba64dc930b01c3eee55ef2b93d/src/DeviceRunners.VisualRunners.Maui/App/Pages/TestAssemblyPage.xaml
106+
FindAppiumElementThenClick("//android.widget.EditText", "All");
107+
await Task.Delay(100);
108+
FindAppiumElementThenClick("//android.widget.TextView", "Failed");
109+
await Task.Delay(500);
110+
111+
StringBuilder sb = new StringBuilder();
112+
sb.AppendLine("PASSED TESTS: " + numPassed + " | FAILED TESTS: " + numFailed);
113+
114+
IReadOnlyCollection<AppiumElement> textResults = driver.FindElements(By.XPath("//android.widget.TextView"));
115+
foreach (var element in textResults)
116+
{
117+
sb.AppendLine(element.Text);
118+
}
119+
120+
Assert.That(numFailed, Is.EqualTo(0), sb.ToString());
121+
}
122+
}
123+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
app: ..\Microsoft.ML.OnnxRuntime.Tests.MAUI\bin\Release\net8.0-android\publish\ORT.CSharp.Tests.MAUI-Signed.apk
2+
platforms:
3+
- platformName: android
4+
deviceName: Samsung Galaxy S22 Ultra
5+
platformVersion: 12.0
6+
browserstackLocal: true
7+
buildName: ORT android test
8+
buildIdentifier: ${BUILD_NUMBER}
9+
projectName: ORT-UITests
10+
debug: true
11+
networkLogs: false
12+
testContextOptions:
13+
skipSessionStatus: true

0 commit comments

Comments
 (0)