[WIP] Automated tests are added to the project

.github/workflows/build-verification.yml

@@ -17,9 +17,13 @@ jobs:
         with:
           java-version: '8'
           distribution: 'adopt'
-      - name: Set up Gradle
-        uses: gradle/actions/setup-gradle@v4
-        with:
-          develocity-access-key: ${{ secrets.DV_SOLUTIONS_ACCESS_KEY }}
+        # @todo the init script from setup-gradle interferes with the BVS tests
+#      - name: Set up Gradle
+#        uses: gradle/actions/setup-gradle@v4
+#        with:
+#          develocity-access-key: ${{ secrets.DV_SOLUTIONS_ACCESS_KEY }}
       - name: Build with Gradle
         run: ./gradlew build
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DV_SOLUTIONS_ACCESS_KEY }}
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.DV_SOLUTIONS_ACCESS_KEY }} # required while injection still uses GE plugin

build.gradle.kts

@@ -319,6 +319,16 @@ tasks.assemble {
     dependsOn(assembleGradleScripts, assembleMavenScripts, assembleLegacyGradleScripts, assembleLegacyMavenScripts)
 }
 
+configurations.consumable("gradleScriptsConsumable") {
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("gradle-build-validation-scripts"))
+    outgoing.artifact(assembleGradleScripts)
+}
+
+configurations.consumable("mavenScriptsConsumable") {
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("maven-build-validation-scripts"))
+    outgoing.artifact(assembleMavenScripts)
+}
+
 val shellcheckGradleScripts by tasks.registering(Shellcheck::class) {
     group = "verification"
     description = "Perform quality checks on Gradle build validation scripts using Shellcheck."

gradle.properties

@@ -4,3 +4,5 @@ org.gradle.parallel=true
 org.gradle.caching=true
 org.gradle.configuration-cache=true
 org.gradle.jvmargs=-Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8
+
+buildValidationTestDevelocityServer=https://ge.solutions-team.gradle.com

settings.gradle.kts

@@ -27,6 +27,7 @@ buildCache {
 rootProject.name = "build-validation-scripts"
 
 include("components/configure-gradle-enterprise-maven-extension")
+include("test")
 
 project(":components/configure-gradle-enterprise-maven-extension").name = "configure-gradle-enterprise-maven-extension"
 

test/build.gradle.kts

@@ -0,0 +1,90 @@
+@file:Suppress("UnstableApiUsage", "HttpUrlsUsage")
+
+plugins {
+    id("groovy")
+    id("jvm-test-suite")
+}
+
+java {
+    toolchain {
+        languageVersion = JavaLanguageVersion.of(8)
+        vendor = JvmVendorSpec.AZUL
+    }
+}
+
+val gradleScripts = configurations.dependencyScope("gradleScripts") {
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("gradle-build-validation-scripts"))
+}.get()
+
+val gradleScriptsResolvable = configurations.resolvable("${gradleScripts.name}Resolvable") {
+    extendsFrom(gradleScripts)
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("gradle-build-validation-scripts"))
+}
+
+val mavenScripts = configurations.dependencyScope("mavenScripts") {
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("maven-build-validation-scripts"))
+}.get()
+
+val mavenScriptsResolvable = configurations.resolvable("${mavenScripts.name}Resolvable") {
+    extendsFrom(mavenScripts)
+    attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named("maven-build-validation-scripts"))
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    gradleScripts(project(":"))
+    mavenScripts(project(":"))
+}
+
+val test by testing.suites.getting(JvmTestSuite::class) {
+    useSpock()
+    dependencies {
+        implementation(gradleTestKit())
+    }
+
+    targets.configureEach {
+        testTask {
+            val develocityKeysFile = gradle.gradleUserHomeDir.resolve("develocity/keys.properties")
+            val develocityKeysEnv = providers.environmentVariable("DEVELOCITY_ACCESS_KEY")
+            val testDevelocityServer = providers.gradleProperty("buildValidationTestDevelocityServer")
+
+            onlyIf("has credentials for Develocity testing server") {
+                val testDevelocityServerHost = testDevelocityServer.get().removePrefix("https://").removePrefix("http://")
+                (develocityKeysFile.exists() && develocityKeysFile.readText().contains(testDevelocityServerHost))
+                        || develocityKeysEnv.map { it.contains(testDevelocityServerHost) }.getOrElse(false)
+            }
+
+            jvmArgumentProviders.add(objects.newInstance<BuildValidationTestConfigurationProvider>().apply {
+                develocityServer = testDevelocityServer
+                jdk8HomeDirectory = javaLauncher.map { it.metadata.installationPath.asFile.absolutePath }
+            })
+        }
+    }
+}
+
+tasks.processTestResources {
+    from(gradleScriptsResolvable)
+    from(mavenScriptsResolvable)
+}
+
+abstract class BuildValidationTestConfigurationProvider : CommandLineArgumentProvider {
+
+    @get:Input
+    abstract val develocityServer: Property<String>
+
+    // JDK version is already an input to the test task.
+    // Its location on disk doesn't matter.
+    @get:Internal
+    abstract val jdk8HomeDirectory: Property<String>
+
+    override fun asArguments(): List<String> {
+        return listOf(
+            "-Dbuild-validation.test.develocity.server=${develocityServer.get()}",
+            "-Dbuild-validation.test.jdk8-home=${jdk8HomeDirectory.get()}"
+        )
+    }
+
+}

test/src/test/groovy/com/gradle/BaseScriptsTest.groovy

@@ -0,0 +1,158 @@
+package com.gradle
+
+import spock.lang.Shared
+import spock.lang.Specification
+import spock.lang.TempDir
+
+import java.nio.file.Files
+import java.util.concurrent.TimeUnit
+
+abstract class BaseScriptsTest extends Specification {
+
+    final static String develocityServer = System.getProperty("build-validation.test.develocity.server")
+    final static String jdk8HomeDirectory = System.getProperty("build-validation.test.jdk8-home")
+
+    @TempDir
+    @Shared
+    private File workingDirectory
+
+    // Common 'where' conditions
+    boolean hasDevelocityConfigured = true
+    boolean hasInvalidServerConfigured = false
+
+    // Test project components
+    @TempDir
+    File testProjectDirectory
+    File gitignore
+
+    // Script arguments
+    String[] tasks = ["build"]
+    String[] goals = ["verify"]
+    private File gitRepo
+
+    // Outcomes
+    int exitCode
+    String output
+
+    void setupSpec() {
+        unpackGradleScripts()
+    }
+
+    private void unpackGradleScripts() {
+        def gradleScriptsResource = new File(this.class.getResource("/develocity-gradle-build-validation-dev.zip").toURI())
+        def gradleScriptsArchive = new File(workingDirectory, "develocity-gradle-build-validation-dev.zip")
+        copy(gradleScriptsResource, gradleScriptsArchive)
+        unzip(gradleScriptsArchive)
+    }
+
+    void setup() {
+        gitignore = new File(testProjectDirectory, ".gitignore")
+        gitRepo = testProjectDirectory
+    }
+
+    String ifDevelocityConfigured(String value) {
+        return hasDevelocityConfigured ? value : ""
+    }
+
+    static enum Experiment {
+        GRADLE_EXP_1("01-validate-incremental-building", "exp1-gradle"),
+        GRADLE_EXP_2("02-validate-local-build-caching-same-location", "exp2-gradle"),
+        GRADLE_EXP_3("03-validate-local-build-caching-different-locations", "exp3-gradle"),
+        MAVEN_EXP_1("01-validate-local-build-caching-same-location", "exp1-maven"),
+        MAVEN_EXP_2("02-validate-local-build-caching-different-locations", "exp2-maven");
+
+        static final List<Experiment> ALL_GRADLE_EXPERIMENTS = [GRADLE_EXP_1, GRADLE_EXP_2, GRADLE_EXP_3]
+        static final List<Experiment> ALL_MAVEN_EXPERIMENTS = [MAVEN_EXP_1, MAVEN_EXP_2]
+
+        private final String scriptName
+        private final String shortName
+
+        Experiment(String scriptName, String shortName) {
+            this.scriptName = scriptName
+            this.shortName = shortName
+        }
+
+        boolean isGradle() {
+            return [GRADLE_EXP_1, GRADLE_EXP_2, GRADLE_EXP_3].contains(this)
+        }
+
+        String getContainingDirectory() {
+            return "develocity-${isGradle() ? "gradle" : "maven"}-build-validation"
+        }
+
+        @Override
+        String toString() {
+            return shortName
+        }
+
+    }
+
+    void run(Experiment experiment, String... args) {
+        buildTestProject()
+        initializeTestProjectRepository()
+
+        String[] command = new String[] {
+            "./${experiment.scriptName}.sh",
+            "--git-repo", "file://${gitRepo.absolutePath}"
+        }
+        command += experiment.isGradle() ? ["--tasks", tasks.join(" ")] : ["--goals", goals.join(" ")]
+        command += args
+        println("\n\$ ${command.join(" ")}")
+
+        def result = runProcess(new File(workingDirectory, experiment.containingDirectory), command)
+        exitCode = result.exitCode
+        output = result.output
+    }
+
+    abstract void buildTestProject()
+
+    private void initializeTestProjectRepository() {
+        runProcess(testProjectDirectory, "git", "init")
+        runProcess(testProjectDirectory, "git", "config", "user.email", "[email protected]")
+        runProcess(testProjectDirectory, "git", "config", "user.name", "Bill D. Tual")
+        runProcess(testProjectDirectory, "git", "add", ".")
+        runProcess(testProjectDirectory, "git", "commit", "-m", "'Create project'")
+    }
+
+    private static ProcessResult runProcess(File workingDirectory, String... args) {
+        def processBuilder = new ProcessBuilder(args).directory(workingDirectory).redirectErrorStream(true)
+        processBuilder.environment()["JAVA_HOME"] = jdk8HomeDirectory
+        def process = processBuilder.start()
+        def output = new StringBuilder()
+        try (def reader = new BufferedReader(new InputStreamReader(process.inputStream))) {
+            reader.eachLine {
+                println(it)
+                output.append(it).append('\n')
+            }
+        }
+        process.waitFor(3, TimeUnit.SECONDS)
+        return new ProcessResult(process.exitValue(), output.toString())
+    }
+
+    private static void copy(File target, File destination) {
+        Files.copy(target.toPath(), destination.toPath())
+    }
+
+    private static void unzip(File target) {
+        runProcess(target.parentFile, "unzip", "-q", "-o", target.name)
+    }
+
+    private static class ProcessResult {
+
+        final int exitCode
+        final String output
+
+        ProcessResult(int exitCode, String output) {
+            this.exitCode = exitCode
+            this.output = output
+        }
+    }
+
+    void scriptCompletesSuccessfullyWithSummary() {
+        assert exitCode == 0
+        assert output.contains("Summary")
+        assert output.contains("Performance Characteristics")
+        assert output.contains("Investigation Quick Links")
+    }
+
+}