Merge "Move to only building APKs that are required for testing" into androidx-main

Author: liutikas

buildSrc/private/src/main/kotlin/androidx/build/AndroidXImplPlugin.kt

@@ -29,15 +29,13 @@ import androidx.build.checkapi.KmpApiTaskConfig
 import androidx.build.checkapi.LibraryApiTaskConfig
 import androidx.build.checkapi.configureProjectForApiTasks
 import androidx.build.dependencies.KOTLIN_VERSION
-import androidx.build.dependencyTracker.AffectedModuleDetector
 import androidx.build.docs.AndroidXKmpDocsImplPlugin
 import androidx.build.gradle.isRoot
 import androidx.build.license.configureExternalDependencyLicenseCheck
 import androidx.build.resources.configurePublicResourcesStub
 import androidx.build.sbom.validateAllArchiveInputsRecognized
 import androidx.build.studio.StudioTask
 import androidx.build.testConfiguration.addAppApkToTestConfigGeneration
-import androidx.build.testConfiguration.addToTestZips
 import androidx.build.testConfiguration.configureTestConfigGeneration
 import com.android.build.api.artifact.SingleArtifact
 import com.android.build.api.dsl.ManagedVirtualDevice
@@ -54,7 +52,6 @@ import com.android.build.gradle.LibraryPlugin
 import com.android.build.gradle.TestExtension
 import com.android.build.gradle.TestPlugin
 import com.android.build.gradle.TestedExtension
-import com.android.build.gradle.internal.tasks.ListingFileRedirectTask
 import java.io.File
 import java.time.Duration
 import java.util.Locale
@@ -702,44 +699,13 @@ class AndroidXImplPlugin @Inject constructor(val componentFactory: SoftwareCompo
         project.configureTestConfigGeneration(this)
         project.configureFtlRunner()
 
-        val buildTestApksTask = project.rootProject.tasks.named(BUILD_TEST_APKS_TASK)
-        when (this) {
-            is TestedExtension -> testVariants
-            // app module defines variants for test module
-            is TestExtension -> applicationVariants
-            else -> throw IllegalStateException("Unsupported plugin type")
-        }.all { variant ->
-            buildTestApksTask.configure {
-                it.dependsOn(variant.assembleProvider)
-            }
-            variant.configureApkZipping(project)
-        }
-
         // AGP warns if we use project.buildDir (or subdirs) for CMake's generated
         // build files (ninja build files, CMakeCache.txt, etc.). Use a staging directory that
         // lives alongside the project's buildDir.
         externalNativeBuild.cmake.buildStagingDirectory =
             File(project.buildDir, "../nativeBuildStaging")
     }
 
-    /**
-     * Configures the ZIP_TEST_CONFIGS_WITH_APKS_TASK to include the test apk if applicable
-     */
-    @Suppress("DEPRECATION") // ApkVariant
-    private fun com.android.build.gradle.api.ApkVariant.configureApkZipping(
-        project: Project
-    ) {
-        packageApplicationProvider.get().let { packageTask ->
-            AffectedModuleDetector.configureTaskGuard(packageTask)
-            addToTestZips(project, packageTask)
-        }
-        // This task needs to be guarded by AffectedModuleDetector due to guarding test
-        // APK building above. It can only be removed if we stop using AMD for test APKs.
-        project.tasks.withType(ListingFileRedirectTask::class.java).forEach {
-            AffectedModuleDetector.configureTaskGuard(it)
-        }
-    }
-
     private fun LibraryExtension.configureAndroidLibraryOptions(
         project: Project,
         androidXExtension: AndroidXExtension
@@ -850,17 +816,6 @@ class AndroidXImplPlugin @Inject constructor(val componentFactory: SoftwareCompo
 
         project.addAppApkToTestConfigGeneration()
         project.addAppApkToFtlRunner()
-
-        val buildTestApksTask = project.rootProject.tasks.named(BUILD_TEST_APKS_TASK)
-        applicationVariants.all { variant ->
-            // Using getName() instead of name due to b/150427408
-            if (variant.buildType.name == "debug") {
-                buildTestApksTask.configure {
-                    it.dependsOn(variant.assembleProvider)
-                }
-            }
-            variant.configureApkZipping(project)
-        }
     }
 
     private fun Project.configureDependencyVerification(
@@ -1001,7 +956,6 @@ class AndroidXImplPlugin @Inject constructor(val componentFactory: SoftwareCompo
     }
 
     companion object {
-        const val BUILD_TEST_APKS_TASK = "buildTestApks"
         const val CREATE_LIBRARY_BUILD_INFO_FILES_TASK = "createLibraryBuildInfoFiles"
         const val GENERATE_TEST_CONFIGURATION_TASK = "GenerateTestConfiguration"
         const val ZIP_TEST_CONFIGS_WITH_APKS_TASK = "zipTestConfigsWithApks"
@@ -1097,14 +1051,6 @@ private fun Project.configureJavaCompilationWarnings(androidXExtension: AndroidX
     }
 }
 
-/**
- * Guarantees unique names for the APKs, and modifies some of the suffixes. The APK name is used
- * to determine what gets run by our test runner
- */
-fun String.renameApkForTesting(projectPath: String): String {
-    return "${projectPath.asFilenamePrefix()}_$this"
-}
-
 fun Project.hasBenchmarkPlugin(): Boolean {
     return this.plugins.hasPlugin(BenchmarkPlugin::class.java)
 }

buildSrc/private/src/main/kotlin/androidx/build/AndroidXRootImplPlugin.kt

@@ -136,8 +136,6 @@ abstract class AndroidXRootImplPlugin : Plugin<Project> {
             }
         }
 
-        tasks.register(AndroidXImplPlugin.BUILD_TEST_APKS_TASK)
-
         // NOTE: this task is used by the Github CI as well. If you make any changes here,
         // please update the .github/workflows files as well, if necessary.
         project.tasks.register(
@@ -150,6 +148,7 @@ abstract class AndroidXRootImplPlugin : Plugin<Project> {
             it.entryCompression = ZipEntryCompression.STORED
             // Archive is greater than 4Gb :O
             it.isZip64 = true
+            it.isReproducibleFileOrder = true
         }
         project.tasks.register(
             ZIP_CONSTRAINED_TEST_CONFIGS_WITH_APKS_TASK, Zip::class.java
@@ -161,6 +160,7 @@ abstract class AndroidXRootImplPlugin : Plugin<Project> {
             it.entryCompression = ZipEntryCompression.STORED
             // Archive is greater than 4Gb :O
             it.isZip64 = true
+            it.isReproducibleFileOrder = true
         }
 
         AffectedModuleDetector.configure(gradle, this)

buildSrc/private/src/main/kotlin/androidx/build/testConfiguration/GenerateMediaTestConfigurationTask.kt

@@ -17,8 +17,6 @@
 package androidx.build.testConfiguration
 
 import androidx.build.dependencyTracker.ProjectSubset
-import androidx.build.renameApkForTesting
-import com.android.build.api.variant.BuiltArtifact
 import com.android.build.api.variant.BuiltArtifacts
 import com.android.build.api.variant.BuiltArtifactsLoader
 import java.io.File
@@ -78,18 +76,6 @@ abstract class GenerateMediaTestConfigurationTask : DefaultTask() {
     @get:Input
     abstract val affectedModuleDetectorSubset: Property<ProjectSubset>
 
-    @get:Input
-    abstract val clientToTPath: Property<String>
-
-    @get:Input
-    abstract val clientPreviousPath: Property<String>
-
-    @get:Input
-    abstract val serviceToTPath: Property<String>
-
-    @get:Input
-    abstract val servicePreviousPath: Property<String>
-
     @get:Input
     abstract val minSdk: Property<Int>
 
@@ -117,39 +103,60 @@ abstract class GenerateMediaTestConfigurationTask : DefaultTask() {
     @get:OutputFile
     abstract val jsonClientToTServiceToTServiceTests: RegularFileProperty
 
+    @get:OutputFile
+    abstract val previousClientApk: RegularFileProperty
+
+    @get:OutputFile
+    abstract val totClientApk: RegularFileProperty
+
+    @get:OutputFile
+    abstract val previousServiceApk: RegularFileProperty
+
+    @get:OutputFile
+    abstract val totServiceApk: RegularFileProperty
+
     @TaskAction
     fun generateAndroidTestZip() {
-        val clientToTApk = resolveApk(clientToTFolder, clientToTLoader)
-        val clientPreviousApk = resolveApk(clientPreviousFolder, clientPreviousLoader)
-        val serviceToTApk = resolveApk(serviceToTFolder, serviceToTLoader)
-        val servicePreviousApk = resolveApk(
-            servicePreviousFolder, servicePreviousLoader
+        val clientToTApk = totClientApk.get().asFile
+        val clientToTSha256 = copyApkAndGetSha256(clientToTFolder, clientToTLoader, clientToTApk)
+        val clientPreviousApk = previousClientApk.get().asFile
+        val clientPreviousSha256 = copyApkAndGetSha256(
+            clientPreviousFolder, clientPreviousLoader, clientPreviousApk
         )
+        val serviceToTApk = totServiceApk.get().asFile
+        val serviceToTSha256 = copyApkAndGetSha256(
+            serviceToTFolder, serviceToTLoader, serviceToTApk
+        )
+        val servicePreviousApk = previousServiceApk.get().asFile
+        val servicePreviousSha256 = copyApkAndGetSha256(
+            servicePreviousFolder, servicePreviousLoader, servicePreviousApk
+        )
+
         writeConfigFileContent(
-            clientApk = clientToTApk,
-            serviceApk = serviceToTApk,
-            clientPath = clientToTPath.get(),
-            servicePath = serviceToTPath.get(),
+            clientApkName = clientToTApk.name,
+            serviceApkName = serviceToTApk.name,
+            clientApkSha256 = clientToTSha256,
+            serviceApkSha256 = serviceToTSha256,
             jsonClientOutputFile = jsonClientToTServiceToTClientTests,
             jsonServiceOutputFile = jsonClientToTServiceToTServiceTests,
             isClientPrevious = false,
             isServicePrevious = false
         )
         writeConfigFileContent(
-            clientApk = clientToTApk,
-            serviceApk = servicePreviousApk,
-            clientPath = clientToTPath.get(),
-            servicePath = servicePreviousPath.get(),
+            clientApkName = clientToTApk.name,
+            serviceApkName = servicePreviousApk.name,
+            clientApkSha256 = clientToTSha256,
+            serviceApkSha256 = servicePreviousSha256,
             jsonClientOutputFile = jsonClientToTServicePreviousClientTests,
             jsonServiceOutputFile = jsonClientToTServicePreviousServiceTests,
             isClientPrevious = false,
             isServicePrevious = true
         )
         writeConfigFileContent(
-            clientApk = clientPreviousApk,
-            serviceApk = serviceToTApk,
-            clientPath = clientPreviousPath.get(),
-            servicePath = serviceToTPath.get(),
+            clientApkName = clientPreviousApk.name,
+            serviceApkName = serviceToTApk.name,
+            clientApkSha256 = clientPreviousSha256,
+            serviceApkSha256 = serviceToTSha256,
             jsonClientOutputFile = jsonClientPreviousServiceToTClientTests,
             jsonServiceOutputFile = jsonClientPreviousServiceToTServiceTests,
             isClientPrevious = true,
@@ -165,26 +172,27 @@ abstract class GenerateMediaTestConfigurationTask : DefaultTask() {
             ?: throw RuntimeException("Cannot load required APK for task: $name")
     }
 
-    private fun BuiltArtifact.resolveName(path: String): String {
-        return outputFile.substringAfterLast("/").renameApkForTesting(path)
+    private fun copyApkAndGetSha256(
+        apkFolder: DirectoryProperty,
+        apkLoader: Property<BuiltArtifactsLoader>,
+        destination: File
+    ): String {
+        val artifacts = apkLoader.get().load(apkFolder.get())
+            ?: throw RuntimeException("Cannot load required APK for task: $name")
+        File(artifacts.elements.single().outputFile).copyTo(destination, overwrite = true)
+        return sha256(destination)
     }
 
     private fun writeConfigFileContent(
-        clientApk: BuiltArtifacts,
-        serviceApk: BuiltArtifacts,
-        clientPath: String,
-        servicePath: String,
+        clientApkName: String,
+        serviceApkName: String,
+        clientApkSha256: String,
+        serviceApkSha256: String,
         jsonClientOutputFile: RegularFileProperty,
         jsonServiceOutputFile: RegularFileProperty,
         isClientPrevious: Boolean,
         isServicePrevious: Boolean,
     ) {
-        val clientBuiltArtifact = clientApk.elements.single()
-        val serviceBuiltArtifact = serviceApk.elements.single()
-        val clientApkName = clientBuiltArtifact.resolveName(clientPath)
-        val clientApkSha256 = sha256(File(clientBuiltArtifact.outputFile))
-        val serviceApkName = serviceBuiltArtifact.resolveName(servicePath)
-        val serviceApkSha256 = sha256(File(serviceBuiltArtifact.outputFile))
         createOrFail(jsonClientOutputFile).writeText(
             buildMediaJson(
                 configName = jsonClientOutputFile.asFile.get().name,

buildSrc/private/src/main/kotlin/androidx/build/testConfiguration/GenerateTestConfigurationTask.kt

@@ -17,15 +17,14 @@
 package androidx.build.testConfiguration
 
 import androidx.build.dependencyTracker.ProjectSubset
-import androidx.build.renameApkForTesting
 import com.android.build.api.variant.BuiltArtifactsLoader
 import java.io.File
 import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.Property
-import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.Internal
@@ -34,14 +33,15 @@ import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.PathSensitive
 import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.TaskAction
+import org.gradle.work.DisableCachingByDefault
 
 /**
  * Writes a configuration file in
  * <a href=https://source.android.com/devices/tech/test_infra/tradefed/testing/through-suite/android-test-structure>AndroidTest.xml</a>
  * format that gets zipped alongside the APKs to be tested.
  * This config gets ingested by Tradefed.
  */
-@CacheableTask
+@DisableCachingByDefault(because = "Doesn't benefit from caching")
 abstract class GenerateTestConfigurationTask : DefaultTask() {
 
     @get:InputFiles
@@ -52,10 +52,6 @@ abstract class GenerateTestConfigurationTask : DefaultTask() {
     @get:Internal
     abstract val appLoader: Property<BuiltArtifactsLoader>
 
-    @get:Input
-    @get:Optional
-    abstract val appProjectPath: Property<String>
-
     @get:InputFiles
     @get:PathSensitive(PathSensitivity.RELATIVE)
     abstract val testFolder: DirectoryProperty
@@ -96,6 +92,18 @@ abstract class GenerateTestConfigurationTask : DefaultTask() {
     @get:OutputFile
     abstract val constrainedOutputXml: RegularFileProperty
 
+    @get:OutputFile
+    abstract val outputTestApk: RegularFileProperty
+
+    @get:OutputFile
+    abstract val constrainedOutputTestApk: RegularFileProperty
+
+    @get:[OutputFile Optional]
+    abstract val outputAppApk: RegularFileProperty
+
+    @get:[OutputFile Optional]
+    abstract val constrainedOutputAppApk: RegularFileProperty
+
     @TaskAction
     fun generateAndroidTestZip() {
         writeConfigFileContent(
@@ -125,9 +133,13 @@ abstract class GenerateTestConfigurationTask : DefaultTask() {
                 ?: throw RuntimeException("Cannot load required APK for task: $name")
             // We don't need to check hasBenchmarkPlugin because benchmarks shouldn't have test apps
             val appApkBuiltArtifact = appApk.elements.single()
-            val appName = appApkBuiltArtifact.outputFile.substringAfterLast("/")
-                .renameApkForTesting(appProjectPath.get())
-            configBuilder.appApkName(appName)
+            val destinationApk = if (isConstrained) {
+                constrainedOutputAppApk.get().asFile
+            } else {
+                outputAppApk.get().asFile
+            }
+            File(appApkBuiltArtifact.outputFile).copyTo(destinationApk, overwrite = true)
+            configBuilder.appApkName(destinationApk.name)
                 .appApkSha256(sha256(File(appApkBuiltArtifact.outputFile)))
         }
         configBuilder.additionalApkKeys(additionalApkKeys.get())
@@ -181,16 +193,25 @@ abstract class GenerateTestConfigurationTask : DefaultTask() {
         val testApk = testLoader.get().load(testFolder.get())
             ?: throw RuntimeException("Cannot load required APK for task: $name")
         val testApkBuiltArtifact = testApk.elements.single()
-        val testName = testApkBuiltArtifact.outputFile
-            .substringAfterLast("/")
-            .renameApkForTesting(testProjectPath.get())
-        configBuilder.testApkName(testName)
+        val destinationApk = if (isConstrained) {
+            constrainedOutputTestApk.get().asFile
+        } else {
+            outputTestApk.get().asFile
+        }
+        File(testApkBuiltArtifact.outputFile).copyTo(destinationApk, overwrite = true)
+        configBuilder.testApkName(destinationApk.name)
             .applicationId(testApk.applicationId)
             .minSdk(minSdk.get().toString())
             .testRunner(testRunner.get())
             .testApkSha256(sha256(File(testApkBuiltArtifact.outputFile)))
         createOrFail(outputFile).writeText(configBuilder.buildXml())
         if (!isConstrained) {
+            if (!outputJson.asFile.get().name.startsWith("_")) {
+                // Prefixing json file names with _ allows us to collocate these files
+                // inside of the androidTest.zip to make fetching them less expensive.
+                throw GradleException("json output file names are expected to use _ prefix to, " +
+                    "currently set to ${outputJson.asFile.get().name}")
+            }
             createOrFail(outputJson).writeText(configBuilder.buildJson())
         }
     }