Merge pull request #21101 from wordpress-mobile/introduce_tests_sharding

Introduce tests sharding
wordpress-mobile · Aug 9, 2024 · 206f726 · 206f726
2 parents 3698bcd + 40986a8
commit 206f726
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 34 deletions.
diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh
@@ -19,7 +19,7 @@ if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then
 fi
 
 echo "--- 🚦 Report Tests Status"
-results_file=$(find "build/instrumented-tests" -type f -name "*.xml" -print -quit)
+results_file="build/instrumented-tests/JUnitReport.xml"
 
 if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then
     annotate_test_failures "$results_file" --slack "build-and-ship"
@@ -28,6 +28,6 @@ else
 fi
 
 echo "--- 🧪 Copying test logs for test collector"
-mkdir buildkite-test-analytics && cp build/instrumented-tests/*/*/*.xml buildkite-test-analytics
+mkdir buildkite-test-analytics && cp -r build/instrumented-tests/matrix_* buildkite-test-analytics
 
 exit $TESTS_EXIT_STATUS
diff --git a/WordPress/build.gradle b/WordPress/build.gradle
@@ -13,6 +13,57 @@ plugins {
     id "com.google.dagger.hilt.android"
     id "org.jetbrains.kotlinx.kover"
     id "com.google.devtools.ksp"
+    id "com.osacky.fladle"
+}
+
+fladle {
+    testTargets = [
+            "notPackage org.wordpress.android.ui.screenshots"
+    ]
+    configs {
+        wordpress {
+            def app = "wordpress"
+            debugApk.set(appPathForVariant(app))
+            instrumentationApk.set(testPathForVariant(app))
+            smartFlankGcsPath.set(pathForSmartFlankGcsPath(app))
+            /* Ignoring (already ignored by JUnit#assume) specific tests on the WordPress app to
+            prevent Fladle/Flank from scenario of executing them on a single shard.
+            This avoidance helps prevent test failures due to overly aggressive sharding that
+            could incur unnecessary costs and tests run failure. */
+            testTargets.set(testTargets.get() + [
+                    "notClass org.wordpress.android.e2e.StatsTests",
+                    "notClass org.wordpress.android.e2e.StatsGranularTabsTest",
+            ])
+        }
+        jetpack {
+            def app = "jetpack"
+            debugApk.set(appPathForVariant(app))
+            instrumentationApk.set(testPathForVariant(app))
+            smartFlankGcsPath.set(pathForSmartFlankGcsPath(app))
+        }
+    }
+    serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json")
+    devices = [
+            ["model": "Pixel2.arm", "version": "30"]
+    ]
+
+    localResultsDir = "$rootDir/build/instrumented-tests"
+
+    maxTestShards = 50
+    shardTime = 120
+}
+
+String appPathForVariant(String app) {
+    String path = "outputs/apk/${app}Vanilla/debug"
+    return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath
+}
+String testPathForVariant(String app) {
+    String path = "outputs/apk/androidTest/${app}Vanilla/debug"
+    return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath
+}
+
+static String pathForSmartFlankGcsPath(String app) {
+    return "gs://fladle/wordpress-android/${app}/smart-flank/JUnitReport.xml"
 }
 
 sentry {

diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-GOOGLE_FIREBASE_SECRETS_PATH = File.join(Dir.home, '.configure', 'wordpress-android', 'secrets', 'firebase.secrets.json')
-
 platform :android do
   #####################################################################################
   # build_and_run_instrumented_test
@@ -19,40 +17,15 @@
 
     gradle(tasks: ["WordPress:assemble#{app.to_s.capitalize}VanillaDebug", "WordPress:assemble#{app.to_s.capitalize}VanillaDebugAndroidTest"])
 
-    # Run the instrumented tests in Firebase Test Lab
-    firebase_login(
-      key_file: GOOGLE_FIREBASE_SECRETS_PATH
-    )
-
-    apk_dir = File.join(PROJECT_ROOT_FOLDER, 'WordPress', 'build', 'outputs', 'apk')
-
-    test_succeeded = android_firebase_test(
-      project_id: firebase_secret(name: 'project_id'),
-      key_file: GOOGLE_FIREBASE_SECRETS_PATH,
-      model: 'Pixel2.arm',
-      version: 30,
-      test_apk_path: File.join(apk_dir, 'androidTest', "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug-androidTest.apk"),
-      apk_path: File.join(apk_dir, "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug.apk"),
-      test_targets: 'notPackage org.wordpress.android.ui.screenshots',
-      results_output_dir: File.join(PROJECT_ROOT_FOLDER, 'build', 'instrumented-tests'),
-      crash_on_test_failure: false
-    )
-
     annotation_ctx = "firebase-test-#{app}-vanilla-debug"
-    if test_succeeded
+    begin
+      gradle(task: "runFlank#{app.to_s.capitalize}")
       sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci?
-    else
-      details_url = lane_context[SharedValues::FIREBASE_TEST_MORE_DETAILS_URL]
-      message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})"
+    rescue StandardError
+      details_url = sh('jq', '.[].webLink', '../build/instrumented-tests/matrix_ids.json', '-r')
+      message = "#{app} Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})"
       sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci?
       UI.test_failure!(message)
     end
   end
 end
-
-def firebase_secret(name:)
-  UI.user_error!('Unable to locale Firebase Secrets File – did you run `configure apply`?') unless File.file? GOOGLE_FIREBASE_SECRETS_PATH
-  key_file_secrets = JSON.parse(File.read(GOOGLE_FIREBASE_SECRETS_PATH))
-  UI.user_error!("Unable to find key `#{name}` in #{GOOGLE_FIREBASE_SECRETS_PATH}") if key_file_secrets[name].nil?
-  key_file_secrets[name]
-end
diff --git a/settings.gradle b/settings.gradle
@@ -11,6 +11,7 @@ pluginManagement {
     gradle.ext.measureBuildsVersion = '3.1.0'
     gradle.ext.koverVersion = '0.7.5'
     gradle.ext.dependencyAnalysisVersion = '1.28.0'
+    gradle.ext.fladleVersion = '0.17.5'
 
     plugins {
         id "org.jetbrains.kotlin.android" version gradle.ext.kotlinVersion
@@ -30,6 +31,7 @@ pluginManagement {
         id "com.google.dagger.hilt.android" version gradle.ext.daggerVersion
         id "com.google.devtools.ksp" version gradle.ext.kspVersion
         id "com.autonomousapps.dependency-analysis" version gradle.ext.dependencyAnalysisVersion
+        id "com.osacky.fladle" version gradle.ext.fladleVersion
     }
     repositories {
         maven {