Skip to content

Commit 2e03393

Browse files
ih0r-dih0r-d
committed
feat: requirements.txt support for maven plugin
1 parent 34f321a commit 2e03393

File tree

3 files changed

+122
-79
lines changed

3 files changed

+122
-79
lines changed

‎graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/AbstractGraalPyMojo.java‎

Lines changed: 63 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,9 @@ public AbstractGraalPyMojo(ProjectBuilder projectBuilder) {
110110
@Parameter
111111
List<String> packages;
112112

113+
@Parameter(property = "requirementsFile", defaultValue = "requirements.txt")
114+
String requirementsFile;
115+
113116
@SuppressFBWarnings("UUF_UNUSED_FIELD")
114117
public static class PythonHome {
115118
@SuppressWarnings("unused")
@@ -124,7 +127,7 @@ public static class PythonHome {
124127
@Parameter(defaultValue = "${session}", readonly = true, required = true)
125128
private MavenSession session;
126129

127-
private ProjectBuilder projectBuilder;
130+
private final ProjectBuilder projectBuilder;
128131

129132
private Set<String> launcherClassPath;
130133

@@ -148,14 +151,26 @@ protected void listGraalPyResources() throws MojoExecutionException {
148151
}
149152
}
150153

151-
protected void preExec(boolean enableWarnings) throws MojoExecutionException {
152-
pythonResourcesDirectory = normalizeEmpty(pythonResourcesDirectory);
153-
externalDirectory = normalizeEmpty(externalDirectory);
154-
resourceDirectory = normalizeEmpty(resourceDirectory);
155-
graalPyLockFile = normalizeEmpty(graalPyLockFile);
156-
packages = packages != null
157-
? packages.stream().filter(p -> p != null && !p.trim().isEmpty()).toList()
158-
: List.of();
154+
protected void preExec(boolean enableWarnings) throws MojoExecutionException {
155+
pythonResourcesDirectory = normalizeEmpty(pythonResourcesDirectory);
156+
externalDirectory = normalizeEmpty(externalDirectory);
157+
resourceDirectory = normalizeEmpty(resourceDirectory);
158+
graalPyLockFile = normalizeEmpty(graalPyLockFile);
159+
Path reqFilePath = resolveReqFile();
160+
if (reqFilePath != null) {
161+
getLog().info("GraalPy requirements file: " + reqFilePath);
162+
if (packages != null) {
163+
throw new MojoExecutionException(
164+
"Cannot use <packages> and <requirementsFile> at the same time. "
165+
+ "New option <requirementsFile> is a replacement for using <packages> with list of inline <package>.");
166+
}
167+
168+
packages = loadRequirementsPackages(reqFilePath);
169+
} else if (packages != null) {
170+
packages = packages.stream()
171+
.filter(p -> p != null && !p.trim().isEmpty())
172+
.toList();
173+
}
159174

160175
if (pythonResourcesDirectory != null) {
161176
if (externalDirectory != null) {
@@ -201,7 +216,38 @@ protected void preExec(boolean enableWarnings) throws MojoExecutionException {
201216
}
202217
}
203218

204-
protected void postExec() throws MojoExecutionException {
219+
private Path resolveReqFile() {
220+
if (requirementsFile == null || requirementsFile.isBlank()) {
221+
return null;
222+
}
223+
224+
Path path = Path.of(requirementsFile);
225+
Path finalPath = path.isAbsolute()
226+
? path
227+
: project.getBasedir().toPath().resolve(path).normalize();
228+
229+
if (Files.exists(finalPath)) {
230+
return finalPath;
231+
}
232+
233+
Path defaultReq = project.getBasedir().toPath().resolve("requirements.txt").normalize();
234+
if (Files.exists(defaultReq)) {
235+
return defaultReq;
236+
}
237+
238+
return null;
239+
}
240+
private List<String> loadRequirementsPackages(Path path) throws MojoExecutionException {
241+
try {
242+
return VFSUtils.requirementsPackages(path);
243+
} catch (IOException e) {
244+
throw new MojoExecutionException(
245+
"Failed to read Python requirements from file: " + requirementsFile
246+
+ ". Please verify that the file exists and is readable.", e);
247+
}
248+
}
249+
250+
protected void postExec() throws MojoExecutionException {
205251
for (Resource r : project.getBuild().getResources()) {
206252
if (Files.exists(Path.of(r.getDirectory(), resourceDirectory, "proj"))) {
207253
getLog().warn(String.format("usage of %s is deprecated, use %s instead",
@@ -249,12 +295,11 @@ private static String normalizeEmpty(String s) {
249295
}
250296

251297
protected Launcher createLauncher() {
252-
Launcher launcherArg = new Launcher(getLauncherPath()) {
253-
public Set<String> computeClassPath() throws IOException {
254-
return calculateLauncherClasspath(project);
255-
}
256-
};
257-
return launcherArg;
298+
return new Launcher(getLauncherPath()) {
299+
public Set<String> computeClassPath() throws IOException {
300+
return calculateLauncherClasspath(project);
301+
}
302+
};
258303
}
259304

260305
protected Path getLockFile() {
@@ -293,7 +338,7 @@ protected static String getGraalPyVersion(MavenProject project) throws IOExcepti
293338

294339
private static Artifact getGraalPyArtifact(MavenProject project) throws IOException {
295340
var projectArtifacts = resolveProjectDependencies(project);
296-
Artifact graalPyArtifact = projectArtifacts.stream().filter(a -> isPythonArtifact(a)).findFirst().orElse(null);
341+
Artifact graalPyArtifact = projectArtifacts.stream().filter(AbstractGraalPyMojo::isPythonArtifact).findFirst().orElse(null);
297342
return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new IOException(
298343
"Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID,
299344
PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID)));
@@ -326,6 +371,7 @@ private Set<String> calculateLauncherClasspath(MavenProject project) throws IOEx
326371
&& PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId()))
327372
.findFirst().orElse(null);
328373
// python-launcher artifact
374+
assert graalPyLauncherArtifact != null;
329375
launcherClassPath.add(graalPyLauncherArtifact.getFile().getAbsolutePath());
330376
// and transitively all its dependencies
331377
launcherClassPath.addAll(resolveDependencies(graalPyLauncherArtifact));

‎org.graalvm.python.embedding.tools/src/main/java/org/graalvm/python/embedding/tools/vfs/VFSUtils.java‎

Lines changed: 30 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
*/
4141
package org.graalvm.python.embedding.tools.vfs;
4242

43+
import java.io.Serial;
4344
import org.graalvm.python.embedding.tools.exec.BuildToolLog;
4445
import org.graalvm.python.embedding.tools.exec.BuildToolLog.CollectOutputLog;
4546
import org.graalvm.python.embedding.tools.exec.GraalPyRunner;
@@ -317,19 +318,17 @@ static InstalledPackages fromVenv(Path venvDirectory) throws IOException {
317318
return new InstalledPackages(venvDirectory, installed, pkgs);
318319
}
319320

320-
List<String> freeze(BuildToolLog log) throws IOException {
321+
void freeze(BuildToolLog log) throws IOException {
321322
CollectOutputLog collectOutputLog = new CollectOutputLog(log);
322323
runPip(venvDirectory, "freeze", collectOutputLog, "--local");
323324
packages = new ArrayList<>(collectOutputLog.getOutput());
324325

325326
String toWrite = "# Generated by GraalPy Maven or Gradle plugin using pip freeze\n"
326327
+ "# This file is used by GraalPy VirtualFileSystem\n" + String.join("\n", packages);
327-
Files.write(installedFile, toWrite.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE,
328+
Files.writeString(installedFile, toWrite, StandardOpenOption.CREATE,
328329
StandardOpenOption.TRUNCATE_EXISTING);
329330

330331
logDebug(log, packages, "VFSUtils venv packages after install %s:", installedFile);
331-
332-
return packages;
333332
}
334333
}
335334

@@ -365,12 +364,10 @@ static VenvContents fromVenv(Path venvDirectory) throws IOException {
365364
if (lines.get(0).startsWith("version=")) {
366365
// this was created with version >= 25
367366
Map<String, String> m = new HashMap<>();
368-
Iterator<String> it = lines.iterator();
369-
while (it.hasNext()) {
370-
String l = it.next();
371-
int idx = l.indexOf("=");
372-
m.put(l.substring(0, idx), l.substring(idx + 1));
373-
}
367+
for (String l : lines) {
368+
int idx = l.indexOf("=");
369+
m.put(l.substring(0, idx), l.substring(idx + 1));
370+
}
374371
String graalPyVersion = m.get(KEY_VERSION);
375372
String platform = m.get(KEY_PLATFORM);
376373
String packagesLine = m.get(KEY_PACKAGES);
@@ -516,7 +513,8 @@ private static List<String> getHeaderList(String lockFileHeader) {
516513
}
517514

518515
public static final class PackagesChangedException extends Exception {
519-
private static final long serialVersionUID = 9162516912727973035L;
516+
@Serial
517+
private static final long serialVersionUID = 9162516912727973035L;
520518

521519
private final transient List<String> pluginPackages;
522520
private final transient List<String> lockFilePackages;
@@ -716,15 +714,16 @@ private static void logPackages(List<String> packages, Path lockFile, BuildToolL
716714
}
717715
}
718716

719-
private static List<String> readPackagesFromFile(Path file) throws IOException {
720-
return Files.readAllLines(file).stream().filter((s) -> {
721-
if (s == null) {
722-
return false;
723-
}
724-
String l = s.trim();
725-
return !l.startsWith("#") && !s.isEmpty();
726-
}).toList();
727-
}
717+
private static List<String> readPackagesFromFile(Path file) throws IOException {
718+
return Files.readAllLines(file).stream()
719+
.map(String::trim)
720+
.filter(line -> !line.isEmpty() && !line.startsWith("#"))
721+
.toList();
722+
}
723+
724+
public static List<String> requirementsPackages(Path requirementsFile) throws IOException {
725+
return Files.exists(requirementsFile) ? readPackagesFromFile(requirementsFile) : Collections.emptyList();
726+
}
728727

729728
private static VenvContents ensureVenv(Path venvDirectory, String graalPyVersion, Launcher launcher,
730729
BuildToolLog log) throws IOException {
@@ -783,7 +782,7 @@ private static boolean install(Path venvDirectory, List<String> newPackages, Ven
783782
private static void missingLockFileWarning(Path venvDirectory, List<String> newPackages,
784783
String missingLockFileWarning, BuildToolLog log) throws IOException {
785784
if (missingLockFileWarning != null && !Boolean.getBoolean("graalpy.vfs.skipMissingLockFileWarning")) {
786-
if (!newPackages.containsAll(InstalledPackages.fromVenv(venvDirectory).packages)) {
785+
if (!new HashSet<>(newPackages).containsAll(InstalledPackages.fromVenv(venvDirectory).packages)) {
787786
if (log.isWarningEnabled()) {
788787
String txt = missingLockFileWarning + "\n";
789788
for (String t : txt.split("\n")) {
@@ -800,7 +799,7 @@ private static void missingLockFileWarning(Path venvDirectory, List<String> newP
800799
private static void checkPluginPackagesInLockFile(List<String> pluginPackages, LockFile lockFile)
801800
throws PackagesChangedException {
802801
if (pluginPackages.size() != lockFile.inputPackages.size()
803-
|| !pluginPackages.containsAll(lockFile.inputPackages)) {
802+
|| !new HashSet<>(pluginPackages).containsAll(lockFile.inputPackages)) {
804803
throw new PackagesChangedException(new ArrayList<>(pluginPackages),
805804
new ArrayList<>(lockFile.inputPackages));
806805
}
@@ -928,7 +927,7 @@ with open(pyvenvcfg, 'w', encoding='utf-8') as f:
928927
private static boolean installWantedPackages(Path venvDirectory, List<String> packages,
929928
List<String> installedPackages, BuildToolLog log) throws IOException {
930929
Set<String> pkgsToInstall = new HashSet<>(packages);
931-
pkgsToInstall.removeAll(installedPackages);
930+
installedPackages.forEach(pkgsToInstall::remove);
932931
if (pkgsToInstall.isEmpty()) {
933932
return false;
934933
}
@@ -948,7 +947,7 @@ private static boolean deleteUnwantedPackages(Path venvDirectory, List<String> p
948947
}
949948
args.add(0, "-y");
950949

951-
runPip(venvDirectory, "uninstall", log, args.toArray(new String[args.size()]));
950+
runPip(venvDirectory, "uninstall", log, args.toArray(new String[0]));
952951
return true;
953952
}
954953

@@ -980,12 +979,11 @@ private static void runVenvBin(Path venvDirectory, String bin, BuildToolLog log,
980979

981980
private static List<String> trim(List<String> l) {
982981
Iterator<String> it = l.iterator();
983-
while (it.hasNext()) {
984-
String p = it.next();
985-
if (p == null || p.trim().isEmpty()) {
986-
it.remove();
987-
}
988-
}
982+
for (String s : l) {
983+
if (s == null || s.trim().isEmpty()) {
984+
it.remove();
985+
}
986+
}
989987
return l;
990988
}
991989

@@ -1000,9 +998,9 @@ private static void info(BuildToolLog log, String txt, Object... args) {
1000998
}
1001999
}
10021000

1003-
private static void lifecycle(BuildToolLog log, String txt, Object... args) {
1001+
private static void lifecycle(BuildToolLog log, Object... args) {
10041002
if (log.isLifecycleEnabled()) {
1005-
log.lifecycle(String.format(txt, args));
1003+
log.lifecycle(String.format("Created GraalPy lock file: %s", args));
10061004
}
10071005
}
10081006

0 commit comments

Comments
 (0)