From 168a994d165eb9dd3b07b805f68f20c89515c98c Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 12:47:06 +0200 Subject: [PATCH 01/22] Add and configure `fladle` Gradle plugin --- WordPress/build.gradle | 18 ++++++++++++++++++ settings.gradle | 2 ++ 2 files changed, 20 insertions(+) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index ee287ab6cf7b..d8cc7939d2ea 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -13,6 +13,24 @@ plugins { id "com.google.dagger.hilt.android" id "org.jetbrains.kotlinx.kover" id "com.google.devtools.ksp" + id "com.osacky.fladle" +} + +fladle { + variant = "vanillaDebug" + serviceAccountCredentials = rootProject.file(".configure-files/firebase.secrets.json") + testTargets = [ + "notPackage org.wordpress.android.ui.screenshots" + ] + devices = [ + ["model": "Pixel2.arm", "version": "30"] + ] + + localResultsDir = "$rootDir/build/instrumented-tests" + + maxTestShards = 50 + shardTime = 120 + smartFlankGcsPath = "gs://fladle/wordpress-android/smart-flank/JUnitReport.xml" } sentry { diff --git a/settings.gradle b/settings.gradle index 987cdf6d6775..23469d254877 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,6 +11,7 @@ pluginManagement { gradle.ext.measureBuildsVersion = '3.1.0' gradle.ext.koverVersion = '0.7.5' gradle.ext.dependencyAnalysisVersion = '1.28.0' + gradle.ext.fladleVersion = '0.17.5' plugins { id "org.jetbrains.kotlin.android" version gradle.ext.kotlinVersion @@ -30,6 +31,7 @@ pluginManagement { id "com.google.dagger.hilt.android" version gradle.ext.daggerVersion id "com.google.devtools.ksp" version gradle.ext.kspVersion id "com.autonomousapps.dependency-analysis" version gradle.ext.dependencyAnalysisVersion + id "com.osacky.fladle" version gradle.ext.fladleVersion } repositories { maven { From ceb5d82bde6783938425de515e19b081a4185e76 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 13:03:13 +0200 Subject: [PATCH 02/22] Run instrumented tests via fladle --- WordPress/build.gradle | 9 ++++++++- fastlane/lanes/test.rb | 19 +------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index d8cc7939d2ea..c91cfa473e6b 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -17,7 +17,14 @@ plugins { } fladle { - variant = "vanillaDebug" + configs { + wordpress { + variant.set("wordpressVanillaDebug") + } + jetpack { + variant.set("jetpackVanillaDebug") + } + } serviceAccountCredentials = rootProject.file(".configure-files/firebase.secrets.json") testTargets = [ "notPackage org.wordpress.android.ui.screenshots" diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 04eaa7b7d65e..c3856aa921b8 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -19,24 +19,7 @@ gradle(tasks: ["WordPress:assemble#{app.to_s.capitalize}VanillaDebug", "WordPress:assemble#{app.to_s.capitalize}VanillaDebugAndroidTest"]) - # Run the instrumented tests in Firebase Test Lab - firebase_login( - key_file: GOOGLE_FIREBASE_SECRETS_PATH - ) - - apk_dir = File.join(PROJECT_ROOT_FOLDER, 'WordPress', 'build', 'outputs', 'apk') - - test_succeeded = android_firebase_test( - project_id: firebase_secret(name: 'project_id'), - key_file: GOOGLE_FIREBASE_SECRETS_PATH, - model: 'Pixel2.arm', - version: 30, - test_apk_path: File.join(apk_dir, 'androidTest', "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug-androidTest.apk"), - apk_path: File.join(apk_dir, "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug.apk"), - test_targets: 'notPackage org.wordpress.android.ui.screenshots', - results_output_dir: File.join(PROJECT_ROOT_FOLDER, 'build', 'instrumented-tests'), - crash_on_test_failure: false - ) + test_succeeded = gradle(task: "runFlank#{app.to_s.capitalize}") annotation_ctx = "firebase-test-#{app}-vanilla-debug" if test_succeeded From e96ee1a962d4d03504eae49456e52e361f5309bb Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 13:04:17 +0200 Subject: [PATCH 03/22] Update `firebase.secrets.json` path Remove unused Ruby code --- WordPress/build.gradle | 2 +- fastlane/lanes/test.rb | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index c91cfa473e6b..199e59c314b0 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -25,7 +25,7 @@ fladle { variant.set("jetpackVanillaDebug") } } - serviceAccountCredentials = rootProject.file(".configure-files/firebase.secrets.json") + serviceAccountCredentials = rootProject.file("~/.configure-files/wordpress-android/secrets/firebase.secrets.json") testTargets = [ "notPackage org.wordpress.android.ui.screenshots" ] diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index c3856aa921b8..c809db130bdc 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -GOOGLE_FIREBASE_SECRETS_PATH = File.join(Dir.home, '.configure', 'wordpress-android', 'secrets', 'firebase.secrets.json') - platform :android do ##################################################################################### # build_and_run_instrumented_test @@ -32,10 +30,3 @@ end end end - -def firebase_secret(name:) - UI.user_error!('Unable to locale Firebase Secrets File โ€“ did you run `configure apply`?') unless File.file? GOOGLE_FIREBASE_SECRETS_PATH - key_file_secrets = JSON.parse(File.read(GOOGLE_FIREBASE_SECRETS_PATH)) - UI.user_error!("Unable to find key `#{name}` in #{GOOGLE_FIREBASE_SECRETS_PATH}") if key_file_secrets[name].nil? - key_file_secrets[name] -end From 99d5c6770c8c95ff8143a7bfd141d2068801dc7c Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 13:26:51 +0200 Subject: [PATCH 04/22] Fix path to secrets --- WordPress/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 199e59c314b0..77894f54def8 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -25,7 +25,7 @@ fladle { variant.set("jetpackVanillaDebug") } } - serviceAccountCredentials = rootProject.file("~/.configure-files/wordpress-android/secrets/firebase.secrets.json") + serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json") testTargets = [ "notPackage org.wordpress.android.ui.screenshots" ] From 9e6afa086606a51f24902ddba2e76334c85c7359 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 10:11:23 +0200 Subject: [PATCH 05/22] Fix setting up paths for debug and instrumentation apks Setting `variant` under `configs` doesn't seem to work: runningcode/fladle#60 --- WordPress/build.gradle | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 77894f54def8..1a207c527312 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -19,10 +19,12 @@ plugins { fladle { configs { wordpress { - variant.set("wordpressVanillaDebug") + debugApk.set(pathForVariant("wordpress", "debugApk")) + instrumentationApk.set(pathForVariant("wordpress", "instrumentationApk")) } jetpack { - variant.set("jetpackVanillaDebug") + debugApk.set(pathForVariant("jetpack", "debugApk")) + instrumentationApk.set(pathForVariant("jetpack", "instrumentationApk")) } } serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json") @@ -40,6 +42,18 @@ fladle { smartFlankGcsPath = "gs://fladle/wordpress-android/smart-flank/JUnitReport.xml" } +String pathForVariant(String app, String type) { + if (!["debugApk", "instrumentationApk"].contains(type)) { + throw new IllegalArgumentException("Type must be 'debugApk' or 'instrumentationApk', but was '$type'") + } + String appPath = "outputs/apk/${app}Vanilla/debug" + String testPath = "outputs/apk/androidTest/${app}Vanilla/debug" + + String path = type == "debugApk" ? appPath : testPath + + return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath +} + sentry { tracingInstrumentation { enabled = true From 3e93f61d3ff05fcab1fece10f2abe433ef884ee5 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 16:53:02 +0200 Subject: [PATCH 06/22] Fix copying test logs for test collector Copy recursively, as now there'll be multiple xml file reports --- .buildkite/commands/run-instrumented-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh index 7f0740490913..03cbfd7e2827 100755 --- a/.buildkite/commands/run-instrumented-tests.sh +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -28,6 +28,6 @@ else fi echo "--- ๐Ÿงช Copying test logs for test collector" -mkdir buildkite-test-analytics && cp build/instrumented-tests/*/*/*.xml buildkite-test-analytics +mkdir buildkite-test-analytics && cp -r build/instrumented-tests/matrix_* buildkite-test-analytics exit $TESTS_EXIT_STATUS From 14ba431419b6284b4596ceac72cbe9e65782e712 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Wed, 31 Jul 2024 17:18:54 +0200 Subject: [PATCH 07/22] Set different paths for smart flank report file As tests between `wordpress` and `jetpack` can differ. A good example is `StatsTests#e2eAllDayStatsLoad` which runs for Jetpack app only. --- WordPress/build.gradle | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 1a207c527312..e24e8b98ee3a 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -19,12 +19,16 @@ plugins { fladle { configs { wordpress { - debugApk.set(pathForVariant("wordpress", "debugApk")) - instrumentationApk.set(pathForVariant("wordpress", "instrumentationApk")) + def app = "wordpress" + debugApk.set(pathForVariant(app, "debugApk")) + instrumentationApk.set(pathForVariant(app, "instrumentationApk")) + smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) } jetpack { - debugApk.set(pathForVariant("jetpack", "debugApk")) - instrumentationApk.set(pathForVariant("jetpack", "instrumentationApk")) + def app = "jetpack" + debugApk.set(pathForVariant(app, "debugApk")) + instrumentationApk.set(pathForVariant(app, "instrumentationApk")) + smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) } } serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json") @@ -39,7 +43,6 @@ fladle { maxTestShards = 50 shardTime = 120 - smartFlankGcsPath = "gs://fladle/wordpress-android/smart-flank/JUnitReport.xml" } String pathForVariant(String app, String type) { @@ -54,6 +57,10 @@ String pathForVariant(String app, String type) { return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath } +static String pathForSmartFlankGcsPath(String app) { + return "gs://fladle/wordpress-android/${app}/smart-flank/JUnitReport.xml" +} + sentry { tracingInstrumentation { enabled = true From d36b4a980c13ca3c1e33324bce62f935b5813faa Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 10:08:50 +0200 Subject: [PATCH 08/22] Make Buildkite annotation work Instead of relying on env variable used by previously used Fastlane Firebase action, we'll parse report generated by fladle/flank --- fastlane/lanes/test.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index c809db130bdc..354dd749a8d2 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -17,16 +17,15 @@ gradle(tasks: ["WordPress:assemble#{app.to_s.capitalize}VanillaDebug", "WordPress:assemble#{app.to_s.capitalize}VanillaDebugAndroidTest"]) - test_succeeded = gradle(task: "runFlank#{app.to_s.capitalize}") - annotation_ctx = "firebase-test-#{app}-vanilla-debug" - if test_succeeded - sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? - else - details_url = lane_context[SharedValues::FIREBASE_TEST_MORE_DETAILS_URL] - message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" - sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? - UI.test_failure!(message) + begin + gradle(task: "runFlank#{app.to_s.capitalize}") + sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? + rescue + details_url = sh('jq ".[].webLink" ~/WordPress-Android/build/instrumented-tests/matrix_ids.json -r') + message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" + sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? + UI.test_failure!(message) end end end From 3d7c8539f035bd703683e81e8419e3df1cf149b5 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 10:12:36 +0200 Subject: [PATCH 09/22] temp: break instrumentation tests to test Buildkite annotations --- .../java/org/wordpress/android/e2e/pages/MySitesPage.kt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt b/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt index ba549724ea11..f89ab119bd7d 100644 --- a/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt +++ b/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt @@ -138,13 +138,6 @@ class MySitesPage { } fun goToStats(): StatsPage { - val statsButton = Espresso.onView( - Matchers.allOf( - ViewMatchers.withText(R.string.stats), - ViewMatchers.withId(R.id.quick_link_item) - ) - ) - WPSupportUtils.clickOn(statsButton) WPSupportUtils.idleFor(4000) WPSupportUtils.dismissJetpackAdIfPresent() WPSupportUtils.waitForElementToBeDisplayedWithoutFailure(R.id.tabLayout) From cd01ef0f7d5be67700456972cf7ef66609d5dea5 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 10:40:08 +0200 Subject: [PATCH 10/22] Fix Ruby formatting in test.rb --- fastlane/lanes/test.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 354dd749a8d2..8ff762135f50 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -19,13 +19,13 @@ annotation_ctx = "firebase-test-#{app}-vanilla-debug" begin - gradle(task: "runFlank#{app.to_s.capitalize}") - sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? - rescue - details_url = sh('jq ".[].webLink" ~/WordPress-Android/build/instrumented-tests/matrix_ids.json -r') - message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" - sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? - UI.test_failure!(message) + gradle(task: "runFlank#{app.to_s.capitalize}") + sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? + rescue + details_url = sh('jq ".[].webLink" ~/WordPress-Android/build/instrumented-tests/matrix_ids.json -r') + message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" + sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? + UI.test_failure!(message) end end end From a462b4df91e0ecf2c303b35fc2c497338dea4c8c Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 11:04:44 +0200 Subject: [PATCH 11/22] Use double asterix wildcard to locate matrix ids --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 8ff762135f50..0aa2da059758 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -22,7 +22,7 @@ gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue - details_url = sh('jq ".[].webLink" ~/WordPress-Android/build/instrumented-tests/matrix_ids.json -r') + details_url = sh('jq ".[].webLink" **/build/instrumented-tests/matrix_ids.json -r') message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) From 48820269ea985c2ed944b0eaf5226602e635dc55 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 11:22:17 +0200 Subject: [PATCH 12/22] Use relative path to locate matrix ids --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 0aa2da059758..f72200c22d10 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -22,7 +22,7 @@ gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue - details_url = sh('jq ".[].webLink" **/build/instrumented-tests/matrix_ids.json -r') + details_url = sh('jq ".[].webLink" ../../build/instrumented-tests/matrix_ids.json -r') message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) From 7c69f45ac8f4b33507b0ddf2e4c96554f21f45ec Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 11:38:15 +0200 Subject: [PATCH 13/22] Fix invalid matrix_ids relative path --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index f72200c22d10..389d41ec4ed6 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -22,7 +22,7 @@ gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue - details_url = sh('jq ".[].webLink" ../../build/instrumented-tests/matrix_ids.json -r') + details_url = sh('jq ".[].webLink" build/instrumented-tests/matrix_ids.json -r') message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) From 9616d525aa6d257c5d4942bd326c71aa8188a2b1 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 11:58:53 +0200 Subject: [PATCH 14/22] Set correct relative path "every action and every plugin's code runs in the root of the project, while all user code from the Fastfile runs inside the ./fastlane directory." https://docs.fastlane.tools/advanced/fastlane/\#directory-behavior --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 389d41ec4ed6..f4497b94787e 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -22,7 +22,7 @@ gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue - details_url = sh('jq ".[].webLink" build/instrumented-tests/matrix_ids.json -r') + details_url = sh('jq ".[].webLink" ../build/instrumented-tests/matrix_ids.json -r') message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) From cc8356450df2c5a52b3d5129e4b98999a297a678 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 12:41:43 +0200 Subject: [PATCH 15/22] Fix path for instrumented-tests result file This should fix annotating test failures. --- .buildkite/commands/run-instrumented-tests.sh | 2 +- fastlane/lanes/test.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh index 03cbfd7e2827..df6ae1e00031 100755 --- a/.buildkite/commands/run-instrumented-tests.sh +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -19,7 +19,7 @@ if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then fi echo "--- ๐Ÿšฆ Report Tests Status" -results_file=$(find "build/instrumented-tests" -type f -name "*.xml" -print -quit) +results_file="build/instrumented-tests/JUnitReport.xml" if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then annotate_test_failures "$results_file" --slack "build-and-ship" diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index f4497b94787e..9dcd94185257 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -23,7 +23,7 @@ sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue details_url = sh('jq ".[].webLink" ../build/instrumented-tests/matrix_ids.json -r') - message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" + message = "#{app} Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) end From c1ef744174fd3d1c8609b6a30c987ae7c6f819a7 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 15:44:15 +0200 Subject: [PATCH 16/22] Revert "temp: break instrumentation tests" This reverts commit 3d7c8539f035bd703683e81e8419e3df1cf149b5. --- .../java/org/wordpress/android/e2e/pages/MySitesPage.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt b/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt index f89ab119bd7d..ba549724ea11 100644 --- a/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt +++ b/WordPress/src/androidTest/java/org/wordpress/android/e2e/pages/MySitesPage.kt @@ -138,6 +138,13 @@ class MySitesPage { } fun goToStats(): StatsPage { + val statsButton = Espresso.onView( + Matchers.allOf( + ViewMatchers.withText(R.string.stats), + ViewMatchers.withId(R.id.quick_link_item) + ) + ) + WPSupportUtils.clickOn(statsButton) WPSupportUtils.idleFor(4000) WPSupportUtils.dismissJetpackAdIfPresent() WPSupportUtils.waitForElementToBeDisplayedWithoutFailure(R.id.tabLayout) From cc4cc3ba08924d0f836b66e8a183bea849ba6b5e Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 17:59:21 +0200 Subject: [PATCH 17/22] Ignore `e2eAllDayStatsLoad` in `wordpress` variant These tests are ignored anyway during a runtime by `assume` method. This doesn't work well with tests sharding though. In case if one of `e2eAllDayStatsLoad` tests is added on its own to a separate shard, Firebase Test Lab will mark the test suite as failed with message: "Some test executions didn't run any test cases. This is usually caused by aggressive sharding (e.g. more shards than test cases), manual sharding errors, or skipping tests. All executions have overhead time, so these may be billable or may count against your quota. A failure outcome was generated to bring your attention to this." --- WordPress/build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index e24e8b98ee3a..473c2dc04561 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -23,6 +23,11 @@ fladle { debugApk.set(pathForVariant(app, "debugApk")) instrumentationApk.set(pathForVariant(app, "instrumentationApk")) smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) + testTargets.set(testTargets.get() + [ + "notClass org.wordpress.android.e2e.StatsTests", + "notClass org.wordpress.android.e2e.StatsGranularTabsTest", + ] + ) } jetpack { def app = "jetpack" From 10c9a743b8050a4af2a3beebd928316cb6b25308 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Thu, 1 Aug 2024 17:57:26 +0200 Subject: [PATCH 18/22] Update order of `testTargets` So the config in `wordpress` can indeed append new exclusions --- WordPress/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 473c2dc04561..0dc24023c5a7 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -17,6 +17,9 @@ plugins { } fladle { + testTargets = [ + "notPackage org.wordpress.android.ui.screenshots" + ] configs { wordpress { def app = "wordpress" @@ -37,9 +40,6 @@ fladle { } } serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json") - testTargets = [ - "notPackage org.wordpress.android.ui.screenshots" - ] devices = [ ["model": "Pixel2.arm", "version": "30"] ] From 997e8087676b9c8110fc676501802c1e823edfc1 Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Fri, 2 Aug 2024 13:00:59 +0200 Subject: [PATCH 19/22] rescue on `StandardError` when instrumentation tests fail --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 9dcd94185257..db6b7b84c028 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -21,7 +21,7 @@ begin gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? - rescue + rescue StandardError details_url = sh('jq ".[].webLink" ../build/instrumented-tests/matrix_ids.json -r') message = "#{app} Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? From e7d901b8f318e810a92fcb0c3745600ccf5fe9ec Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Fri, 2 Aug 2024 13:03:36 +0200 Subject: [PATCH 20/22] Break down `jq` command and its arguments --- fastlane/lanes/test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index db6b7b84c028..0dc3c4acfd90 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -22,7 +22,7 @@ gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? rescue StandardError - details_url = sh('jq ".[].webLink" ../build/instrumented-tests/matrix_ids.json -r') + details_url = sh('jq', '.[].webLink', '../build/instrumented-tests/matrix_ids.json', '-r') message = "#{app} Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) From 4d7c0b72bbceedafafcaefb3cd1990de2663fa3d Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 5 Aug 2024 11:24:22 +0200 Subject: [PATCH 21/22] Split `pathForVariant` into two methods To increase readability --- WordPress/build.gradle | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 0dc24023c5a7..90cca7d91c46 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -23,19 +23,18 @@ fladle { configs { wordpress { def app = "wordpress" - debugApk.set(pathForVariant(app, "debugApk")) - instrumentationApk.set(pathForVariant(app, "instrumentationApk")) + debugApk.set(appPathForVariant(app)) + instrumentationApk.set(testPathForVariant(app)) smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) testTargets.set(testTargets.get() + [ "notClass org.wordpress.android.e2e.StatsTests", "notClass org.wordpress.android.e2e.StatsGranularTabsTest", - ] - ) + ]) } jetpack { def app = "jetpack" - debugApk.set(pathForVariant(app, "debugApk")) - instrumentationApk.set(pathForVariant(app, "instrumentationApk")) + debugApk.set(appPathForVariant(app)) + instrumentationApk.set(testPathForVariant(app)) smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) } } @@ -50,15 +49,12 @@ fladle { shardTime = 120 } -String pathForVariant(String app, String type) { - if (!["debugApk", "instrumentationApk"].contains(type)) { - throw new IllegalArgumentException("Type must be 'debugApk' or 'instrumentationApk', but was '$type'") - } - String appPath = "outputs/apk/${app}Vanilla/debug" - String testPath = "outputs/apk/androidTest/${app}Vanilla/debug" - - String path = type == "debugApk" ? appPath : testPath - +String appPathForVariant(String app) { + String path = "outputs/apk/${app}Vanilla/debug" + return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath +} +String testPathForVariant(String app) { + String path = "outputs/apk/androidTest/${app}Vanilla/debug" return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath } From 9c05ddd98f2b814921613ea56dbf3d57cc60ca2f Mon Sep 17 00:00:00 2001 From: Wojtek Zieba Date: Mon, 5 Aug 2024 11:30:34 +0200 Subject: [PATCH 22/22] Add a comment about ignoring specific tests on WordPress variant when using Fladle/Flank --- WordPress/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 90cca7d91c46..0c208961692a 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -26,6 +26,10 @@ fladle { debugApk.set(appPathForVariant(app)) instrumentationApk.set(testPathForVariant(app)) smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) + /* Ignoring (already ignored by JUnit#assume) specific tests on the WordPress app to + prevent Fladle/Flank from scenario of executing them on a single shard. + This avoidance helps prevent test failures due to overly aggressive sharding that + could incur unnecessary costs and tests run failure. */ testTargets.set(testTargets.get() + [ "notClass org.wordpress.android.e2e.StatsTests", "notClass org.wordpress.android.e2e.StatsGranularTabsTest",