diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh index 7f0740490913..df6ae1e00031 100755 --- a/.buildkite/commands/run-instrumented-tests.sh +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -19,7 +19,7 @@ if [[ "$TESTS_EXIT_STATUS" -ne 0 ]]; then fi echo "--- ๐Ÿšฆ Report Tests Status" -results_file=$(find "build/instrumented-tests" -type f -name "*.xml" -print -quit) +results_file="build/instrumented-tests/JUnitReport.xml" if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then annotate_test_failures "$results_file" --slack "build-and-ship" @@ -28,6 +28,6 @@ else fi echo "--- ๐Ÿงช Copying test logs for test collector" -mkdir buildkite-test-analytics && cp build/instrumented-tests/*/*/*.xml buildkite-test-analytics +mkdir buildkite-test-analytics && cp -r build/instrumented-tests/matrix_* buildkite-test-analytics exit $TESTS_EXIT_STATUS diff --git a/WordPress/build.gradle b/WordPress/build.gradle index 86056989d7b1..b56a0d066731 100644 --- a/WordPress/build.gradle +++ b/WordPress/build.gradle @@ -13,6 +13,57 @@ plugins { id "com.google.dagger.hilt.android" id "org.jetbrains.kotlinx.kover" id "com.google.devtools.ksp" + id "com.osacky.fladle" +} + +fladle { + testTargets = [ + "notPackage org.wordpress.android.ui.screenshots" + ] + configs { + wordpress { + def app = "wordpress" + debugApk.set(appPathForVariant(app)) + instrumentationApk.set(testPathForVariant(app)) + smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) + /* Ignoring (already ignored by JUnit#assume) specific tests on the WordPress app to + prevent Fladle/Flank from scenario of executing them on a single shard. + This avoidance helps prevent test failures due to overly aggressive sharding that + could incur unnecessary costs and tests run failure. */ + testTargets.set(testTargets.get() + [ + "notClass org.wordpress.android.e2e.StatsTests", + "notClass org.wordpress.android.e2e.StatsGranularTabsTest", + ]) + } + jetpack { + def app = "jetpack" + debugApk.set(appPathForVariant(app)) + instrumentationApk.set(testPathForVariant(app)) + smartFlankGcsPath.set(pathForSmartFlankGcsPath(app)) + } + } + serviceAccountCredentials = new File(System.getProperty("user.home"), ".configure/wordpress-android/secrets/firebase.secrets.json") + devices = [ + ["model": "Pixel2.arm", "version": "30"] + ] + + localResultsDir = "$rootDir/build/instrumented-tests" + + maxTestShards = 50 + shardTime = 120 +} + +String appPathForVariant(String app) { + String path = "outputs/apk/${app}Vanilla/debug" + return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath +} +String testPathForVariant(String app) { + String path = "outputs/apk/androidTest/${app}Vanilla/debug" + return new File(layout.buildDirectory.dir(path).get().asFile, '*.apk').absolutePath +} + +static String pathForSmartFlankGcsPath(String app) { + return "gs://fladle/wordpress-android/${app}/smart-flank/JUnitReport.xml" } sentry { diff --git a/fastlane/lanes/test.rb b/fastlane/lanes/test.rb index 04eaa7b7d65e..0dc3c4acfd90 100644 --- a/fastlane/lanes/test.rb +++ b/fastlane/lanes/test.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -GOOGLE_FIREBASE_SECRETS_PATH = File.join(Dir.home, '.configure', 'wordpress-android', 'secrets', 'firebase.secrets.json') - platform :android do ##################################################################################### # build_and_run_instrumented_test @@ -19,40 +17,15 @@ gradle(tasks: ["WordPress:assemble#{app.to_s.capitalize}VanillaDebug", "WordPress:assemble#{app.to_s.capitalize}VanillaDebugAndroidTest"]) - # Run the instrumented tests in Firebase Test Lab - firebase_login( - key_file: GOOGLE_FIREBASE_SECRETS_PATH - ) - - apk_dir = File.join(PROJECT_ROOT_FOLDER, 'WordPress', 'build', 'outputs', 'apk') - - test_succeeded = android_firebase_test( - project_id: firebase_secret(name: 'project_id'), - key_file: GOOGLE_FIREBASE_SECRETS_PATH, - model: 'Pixel2.arm', - version: 30, - test_apk_path: File.join(apk_dir, 'androidTest', "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug-androidTest.apk"), - apk_path: File.join(apk_dir, "#{app}Vanilla", 'debug', "org.wordpress.android-#{app}-vanilla-debug.apk"), - test_targets: 'notPackage org.wordpress.android.ui.screenshots', - results_output_dir: File.join(PROJECT_ROOT_FOLDER, 'build', 'instrumented-tests'), - crash_on_test_failure: false - ) - annotation_ctx = "firebase-test-#{app}-vanilla-debug" - if test_succeeded + begin + gradle(task: "runFlank#{app.to_s.capitalize}") sh("buildkite-agent annotation remove --context '#{annotation_ctx}' || true") if is_ci? - else - details_url = lane_context[SharedValues::FIREBASE_TEST_MORE_DETAILS_URL] - message = "Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" + rescue StandardError + details_url = sh('jq', '.[].webLink', '../build/instrumented-tests/matrix_ids.json', '-r') + message = "#{app} Firebase Tests failed. Failure details can be seen [here in Firebase Console](#{details_url})" sh('buildkite-agent', 'annotate', message, '--style', 'error', '--context', annotation_ctx) if is_ci? UI.test_failure!(message) end end end - -def firebase_secret(name:) - UI.user_error!('Unable to locale Firebase Secrets File โ€“ did you run `configure apply`?') unless File.file? GOOGLE_FIREBASE_SECRETS_PATH - key_file_secrets = JSON.parse(File.read(GOOGLE_FIREBASE_SECRETS_PATH)) - UI.user_error!("Unable to find key `#{name}` in #{GOOGLE_FIREBASE_SECRETS_PATH}") if key_file_secrets[name].nil? - key_file_secrets[name] -end diff --git a/settings.gradle b/settings.gradle index 72138f0bc54c..e058e4c13cae 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,6 +11,7 @@ pluginManagement { gradle.ext.measureBuildsVersion = '3.1.0' gradle.ext.koverVersion = '0.7.5' gradle.ext.dependencyAnalysisVersion = '1.28.0' + gradle.ext.fladleVersion = '0.17.5' plugins { id "org.jetbrains.kotlin.android" version gradle.ext.kotlinVersion @@ -30,6 +31,7 @@ pluginManagement { id "com.google.dagger.hilt.android" version gradle.ext.daggerVersion id "com.google.devtools.ksp" version gradle.ext.kspVersion id "com.autonomousapps.dependency-analysis" version gradle.ext.dependencyAnalysisVersion + id "com.osacky.fladle" version gradle.ext.fladleVersion } repositories { maven {