Skip to content

Commit 9197c0b

Browse files
linabutlerlinabutler
committed
nimbus-gradle-plugin: Overhaul the plugin to support lazy configuration.
This is a substantial refactor of the plugin that should fix issues like bug 1854254 and bug 1856461, and speed up FML generation. These bugs had the same underlying cause: the plugin was trying to configure its tasks without observing the Gradle build lifecycle, leading to inconsistencies and accidental interdependencies. Further, because the old tasks didn't declare their inputs or outputs, Gradle couldn't "see" the full task graph, causing too much or too little to be rebuilt. The easiest way to force Gradle to pick up the changes used to be a full clobber. This commit resolves all those issues by: * Adopting Gradle's lazy configuration API [1], which lets Gradle track inputs and outputs, and avoids ordering issues caused by realizing the task graph at the wrong time [2]. As a bonus, we should be able to work much better with `afterEvaluate` [3] in m-c's Gradle scripts. * Adopting the new Android Gradle Plugin `SourceDirectories` API that supports lazy configuration [4], which replaces the deprecated `registerJavaGeneratingTask` API. * Adding task classes and custom types to help with naming and configuring inputs and outputs. [1]: https://docs.gradle.org/current/userguide/lazy_configuration.html [2]: https://docs.gradle.org/current/userguide/task_configuration_avoidance.html [3]: https://mbonnin.hashnode.dev/my-life-after-afterevaluate [4]: https://developer.android.com/reference/tools/gradle-api/8.4/com/android/build/api/variant/SourceDirectories
1 parent 574be82 commit 9197c0b

File tree

5 files changed

+507
-273
lines changed

5 files changed

+507
-273
lines changed

‎tools/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusAssembleToolsTask.groovy‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/* This Source Code Form is subject to the terms of the Mozilla Public
2+
* License, v. 2.0. If a copy of the MPL was not distributed with this
3+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4+
5+
package org.mozilla.appservices.tooling.nimbus
6+
7+
import org.gradle.api.Action
8+
import org.gradle.api.DefaultTask
9+
import org.gradle.api.GradleException
10+
import org.gradle.api.file.ArchiveOperations
11+
import org.gradle.api.file.FileVisitDetails
12+
import org.gradle.api.file.RegularFileProperty
13+
import org.gradle.api.model.ObjectFactory
14+
import org.gradle.api.provider.ListProperty
15+
import org.gradle.api.provider.Property
16+
import org.gradle.api.tasks.CacheableTask
17+
import org.gradle.api.tasks.Input
18+
import org.gradle.api.tasks.LocalState
19+
import org.gradle.api.tasks.Nested
20+
import org.gradle.api.tasks.OutputFile
21+
import org.gradle.api.tasks.TaskAction
22+
23+
import javax.inject.Inject
24+
25+
import groovy.transform.Immutable
26+
27+
/**
28+
* A task that fetches a prebuilt `nimbus-fml` binary for the current platform.
29+
*
30+
* Prebuilt binaries for all platforms are packaged into ZIP archives, and
31+
* published to sources like `archive.mozilla.org` (for releases) or
32+
* TaskCluster (for nightly builds).
33+
*
34+
* This task takes a variable number of inputs: a list of archive sources,
35+
* and a list of glob patterns to find the binary for the current platform
36+
* in the archive.
37+
*
38+
* The unzipped binary is this task's only output. This output is then used as
39+
* an optional input to the `NimbusFmlCommandTask`s.
40+
*/
41+
@CacheableTask
42+
abstract class NimbusAssembleToolsTask extends DefaultTask {
43+
@Inject
44+
abstract ArchiveOperations getArchiveOperations()
45+
46+
@Nested
47+
abstract FetchSpec getFetchSpec()
48+
49+
@Nested
50+
abstract UnzipSpec getUnzipSpec()
51+
52+
/** The location of the fetched ZIP archive. */
53+
@LocalState
54+
abstract RegularFileProperty getArchiveFile()
55+
56+
/**
57+
* The location of the fetched hash file, which contains the
58+
* archive's checksum.
59+
*/
60+
@LocalState
61+
abstract RegularFileProperty getHashFile()
62+
63+
/** The location of the unzipped binary. */
64+
@OutputFile
65+
abstract RegularFileProperty getFmlBinary()
66+
67+
/**
68+
* Configures the task to download the archive.
69+
*
70+
* @param action The configuration action.
71+
*/
72+
void fetch(Action<FetchSpec> action) {
73+
action.execute(fetchSpec)
74+
}
75+
76+
/**
77+
* Configures the task to extract the binary from the archive.
78+
*
79+
* @param action The configuration action.
80+
*/
81+
void unzip(Action<UnzipSpec> action) {
82+
action.execute(unzipSpec)
83+
}
84+
85+
@TaskAction
86+
void assembleTools() {
87+
def sources = [fetchSpec, *fetchSpec.fallbackSources.get()].collect {
88+
new Source(new URI(it.archive.get()), new URI(it.hash.get()))
89+
}
90+
91+
def successfulSource = sources.find { it.trySaveArchiveTo(archiveFile.get().asFile) }
92+
if (successfulSource == null) {
93+
throw new GradleException("Couldn't fetch archive from any of: ${sources*.archiveURI.collect { "`$it`" }.join(', ')}")
94+
}
95+
96+
// We get the checksum, although don't do anything with it yet;
97+
// Checking it here would be able to detect if the zip file was tampered with
98+
// in transit between here and the server.
99+
// It won't detect compromise of the CI server.
100+
try {
101+
successfulSource.saveHashTo(hashFile.get().asFile)
102+
} catch (IOException e) {
103+
throw new GradleException("Couldn't fetch hash from `${successfulSource.hashURI}`", e)
104+
}
105+
106+
def zipTree = archiveOperations.zipTree(archiveFile.get())
107+
def visitedFilePaths = []
108+
zipTree.matching {
109+
include unzipSpec.includePatterns.get()
110+
}.visit { FileVisitDetails details ->
111+
if (!details.directory) {
112+
if (visitedFilePaths.empty) {
113+
details.copyTo(fmlBinary.get().asFile)
114+
fmlBinary.get().asFile.setExecutable(true)
115+
}
116+
visitedFilePaths.add(details.relativePath)
117+
}
118+
}
119+
120+
if (visitedFilePaths.size() > 1) {
121+
throw new GradleException("Ambiguous unzip spec matched ${visitedFilePaths.size()} files in archive: ${visitedFilePaths.collect { "`$it`" }.join(', ')}")
122+
}
123+
}
124+
125+
/**
126+
* Specifies the source from which to fetch the archive and
127+
* its hash file.
128+
*/
129+
static abstract class FetchSpec extends SourceSpec {
130+
@Inject
131+
abstract ObjectFactory getObjectFactory()
132+
133+
@Nested
134+
abstract ListProperty<SourceSpec> getFallbackSources()
135+
136+
/**
137+
* Configures a fallback to try if the archive can't be fetched
138+
* from this source.
139+
*
140+
* The task will try fallbacks in the order in which they're
141+
* configured.
142+
*
143+
* @param action The configuration action.
144+
*/
145+
void fallback(Action<SourceSpec> action) {
146+
def spec = objectFactory.newInstance(SourceSpec)
147+
action(spec)
148+
fallbackSources.add(spec)
149+
}
150+
}
151+
152+
/** Specifies the URL of an archive and its hash file. */
153+
static abstract class SourceSpec {
154+
@Input
155+
abstract Property<String> getArchive()
156+
157+
@Input
158+
abstract Property<String> getHash()
159+
}
160+
161+
/**
162+
* Specifies which binary to extract from the fetched archive.
163+
*
164+
* The spec should only match one file in the archive. If the spec
165+
* matches multiple files in the archive, the task will fail.
166+
*/
167+
static abstract class UnzipSpec {
168+
@Input
169+
abstract ListProperty<String> getIncludePatterns()
170+
171+
/**
172+
* Includes all files whose paths match the pattern.
173+
*
174+
* @param pattern An Ant-style glob pattern.
175+
* @see org.gradle.api.tasks.util.PatternFilterable#include
176+
*/
177+
void include(String pattern) {
178+
includePatterns.add(pattern)
179+
}
180+
}
181+
182+
/** A helper to fetch an archive and its hash file. */
183+
@Immutable
184+
static class Source {
185+
URI archiveURI
186+
URI hashURI
187+
188+
boolean trySaveArchiveTo(File destination) {
189+
try {
190+
saveURITo(archiveURI, destination)
191+
true
192+
} catch (IOException ignored) {
193+
false
194+
}
195+
}
196+
197+
void saveHashTo(File destination) {
198+
saveURITo(hashURI, destination)
199+
}
200+
201+
private static void saveURITo(URI source, File destination) {
202+
source.toURL().withInputStream { from ->
203+
destination.withOutputStream { out ->
204+
out << from
205+
}
206+
}
207+
}
208+
}
209+
}

‎tools/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusFeaturesTask.groovy‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/* This Source Code Form is subject to the terms of the Mozilla Public
2+
* License, v. 2.0. If a copy of the MPL was not distributed with this
3+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4+
5+
package org.mozilla.appservices.tooling.nimbus
6+
7+
import org.gradle.api.file.ConfigurableFileCollection
8+
import org.gradle.api.file.DirectoryProperty
9+
import org.gradle.api.file.RegularFileProperty
10+
import org.gradle.api.provider.Property
11+
import org.gradle.api.tasks.CacheableTask
12+
import org.gradle.api.tasks.Input
13+
import org.gradle.api.tasks.InputFile
14+
import org.gradle.api.tasks.InputFiles
15+
import org.gradle.api.tasks.LocalState
16+
import org.gradle.api.tasks.OutputDirectory
17+
import org.gradle.api.tasks.PathSensitive
18+
import org.gradle.api.tasks.PathSensitivity
19+
import org.gradle.process.ExecSpec
20+
21+
@CacheableTask
22+
abstract class NimbusFeaturesTask extends NimbusFmlCommandTask {
23+
@InputFile
24+
@PathSensitive(PathSensitivity.RELATIVE)
25+
abstract RegularFileProperty getInputFile()
26+
27+
@InputFiles
28+
@PathSensitive(PathSensitivity.RELATIVE)
29+
abstract ConfigurableFileCollection getRepoFiles()
30+
31+
@Input
32+
abstract Property<String> getChannel()
33+
34+
@LocalState
35+
abstract DirectoryProperty getCacheDir()
36+
37+
@OutputDirectory
38+
abstract DirectoryProperty getOutputDir()
39+
40+
@Override
41+
void configureFmlCommand(ExecSpec spec) {
42+
spec.with {
43+
args 'generate'
44+
45+
args '--language', 'kotlin'
46+
args '--channel', channel.get()
47+
args '--cache-dir', cacheDir.get()
48+
for (File file : repoFiles) {
49+
args '--repo-file', file
50+
}
51+
52+
args inputFile.get().asFile
53+
args outputDir.get().asFile
54+
}
55+
}
56+
}

‎tools/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusFmlCommandTask.groovy‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/* This Source Code Form is subject to the terms of the Mozilla Public
2+
* License, v. 2.0. If a copy of the MPL was not distributed with this
3+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4+
5+
package org.mozilla.appservices.tooling.nimbus
6+
7+
import org.gradle.api.DefaultTask
8+
import org.gradle.api.GradleException
9+
import org.gradle.api.file.ProjectLayout
10+
import org.gradle.api.file.RegularFileProperty
11+
import org.gradle.api.provider.Property
12+
import org.gradle.api.tasks.Input
13+
import org.gradle.api.tasks.InputFiles
14+
import org.gradle.api.tasks.Optional
15+
import org.gradle.api.tasks.PathSensitive
16+
import org.gradle.api.tasks.PathSensitivity
17+
import org.gradle.api.tasks.TaskAction
18+
import org.gradle.process.ExecOperations
19+
import org.gradle.process.ExecSpec
20+
21+
import javax.inject.Inject
22+
23+
/**
24+
* A base task to execute a `nimbus-fml` command.
25+
*
26+
* Subclasses can declare additional inputs and outputs, and override
27+
* `configureFmlCommand` to set additional command arguments.
28+
*
29+
* This task requires either `applicationServicesDir` to be set, or
30+
* the `fmlBinary` to exist. If `applicationServicesDir` is set,
31+
* the task will run `nimbus-fml` from the Application Services repo;
32+
* otherwise, it'll fall back to a prebuilt `fmlBinary`.
33+
*/
34+
abstract class NimbusFmlCommandTask extends DefaultTask {
35+
public static final String APPSERVICES_FML_HOME = 'components/support/nimbus-fml'
36+
37+
@Inject
38+
abstract ExecOperations getExecOperations()
39+
40+
@Inject
41+
abstract ProjectLayout getProjectLayout()
42+
43+
@Input
44+
abstract Property<String> getProjectDir()
45+
46+
@Input
47+
@Optional
48+
abstract Property<String> getApplicationServicesDir()
49+
50+
// `@InputFiles` instead of `@InputFile` because we don't want
51+
// the task to fail if the `fmlBinary` file doesn't exist
52+
// (https://github.com/gradle/gradle/issues/2016).
53+
@InputFiles
54+
@PathSensitive(PathSensitivity.NONE)
55+
abstract RegularFileProperty getFmlBinary()
56+
57+
/**
58+
* Configures the `nimbus-fml` command for this task.
59+
*
60+
* This method is invoked from the `@TaskAction` during the execution phase,
61+
* and so has access to the final values of the inputs and outputs.
62+
*
63+
* @param spec The specification for the `nimbus-fml` command.
64+
*/
65+
abstract void configureFmlCommand(ExecSpec spec)
66+
67+
@TaskAction
68+
void execute() {
69+
execOperations.exec { spec ->
70+
spec.with {
71+
// Absolutize `projectDir`, so that we can resolve our paths
72+
// against it. If it's already absolute, it'll be used as-is.
73+
def projectDir = projectLayout.projectDirectory.dir(projectDir.get())
74+
def localAppServices = applicationServicesDir.getOrNull()
75+
if (localAppServices == null) {
76+
if (!fmlBinary.get().asFile.exists()) {
77+
throw new GradleException("`nimbus-fml` wasn't downloaded and `nimbus.applicationServicesDir` isn't set")
78+
}
79+
workingDir projectDir
80+
commandLine fmlBinary.get().asFile
81+
} else {
82+
def cargoManifest = projectDir.file("$localAppServices/$APPSERVICES_FML_HOME/Cargo.toml").asFile
83+
84+
commandLine 'cargo'
85+
args 'run'
86+
args '--manifest-path', cargoManifest
87+
args '--'
88+
}
89+
}
90+
configureFmlCommand(spec)
91+
}
92+
}
93+
}

0 commit comments

Comments
 (0)