|
| 1 | +/* |
| 2 | + * Copyright (c) 2022 Microsoft Corporation |
| 3 | + * |
| 4 | + * This program and the accompanying materials are made available under the |
| 5 | + * terms of the Apache License, Version 2.0 which is available at |
| 6 | + * https://www.apache.org/licenses/LICENSE-2.0 |
| 7 | + * |
| 8 | + * SPDX-License-Identifier: Apache-2.0 |
| 9 | + * |
| 10 | + * Contributors: |
| 11 | + * Microsoft Corporation - initial API and implementation |
| 12 | + * |
| 13 | + */ |
| 14 | + |
| 15 | +package org.eclipse.edc.plugins.autodoc.tasks; |
| 16 | + |
| 17 | +import org.eclipse.edc.plugins.autodoc.AutodocExtension; |
| 18 | +import org.gradle.api.DefaultTask; |
| 19 | +import org.gradle.api.artifacts.Dependency; |
| 20 | +import org.gradle.api.artifacts.repositories.MavenArtifactRepository; |
| 21 | +import org.gradle.api.tasks.TaskAction; |
| 22 | + |
| 23 | +import java.io.FileOutputStream; |
| 24 | +import java.io.IOException; |
| 25 | +import java.net.URI; |
| 26 | +import java.net.http.HttpClient; |
| 27 | +import java.net.http.HttpRequest; |
| 28 | +import java.net.http.HttpResponse; |
| 29 | +import java.nio.file.Files; |
| 30 | +import java.nio.file.Path; |
| 31 | +import java.time.Duration; |
| 32 | +import java.time.Instant; |
| 33 | +import java.util.Objects; |
| 34 | +import java.util.Optional; |
| 35 | +import java.util.Set; |
| 36 | + |
| 37 | +import static java.lang.String.format; |
| 38 | +import static java.util.Objects.requireNonNull; |
| 39 | + |
| 40 | +public class ManifestDownloadTask extends DefaultTask { |
| 41 | + |
| 42 | + public static final String NAME = "downloadManifests"; |
| 43 | + private static final String EDC_GROUP = "org.eclipse.edc"; |
| 44 | + private static final Duration MAX_MANIFEST_AGE = Duration.ofHours(24); |
| 45 | + private static final String MANIFEST_CLASSIFIER = "manifest"; |
| 46 | + private static final String MANIFEST_TYPE = "json"; |
| 47 | + private final HttpClient httpClient; |
| 48 | + private Path downloadDirectory; |
| 49 | + |
| 50 | + public ManifestDownloadTask() { |
| 51 | + httpClient = HttpClient.newHttpClient(); |
| 52 | + downloadDirectory = getProject().getRootProject().getBuildDir().toPath().resolve("manifests"); |
| 53 | + } |
| 54 | + |
| 55 | + @TaskAction |
| 56 | + public void downloadManifests() { |
| 57 | + var autodocExt = getProject().getExtensions().findByType(AutodocExtension.class); |
| 58 | + requireNonNull(autodocExt, "AutodocExtension cannot be null"); |
| 59 | + |
| 60 | + if (autodocExt.getDownloadDirectory().isPresent()) { |
| 61 | + downloadDirectory = autodocExt.getDownloadDirectory().get().toPath(); |
| 62 | + } |
| 63 | + |
| 64 | + getProject().getConfigurations() |
| 65 | + .stream().flatMap(config -> config.getDependencies().stream()) |
| 66 | + .filter(dep -> EDC_GROUP.equals(dep.getGroup())) |
| 67 | + .filter(dep -> !getExclusions().contains(dep.getName())) |
| 68 | + .map(this::createDownloadRequest) |
| 69 | + .filter(Optional::isPresent) |
| 70 | + .forEach(dt -> downloadDependency(dt.get(), downloadDirectory)); |
| 71 | + } |
| 72 | + |
| 73 | + private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) { |
| 74 | + return format("%s%s/%s/%s/%s-%s-%s.%s", repo.getUrl(), dep.getGroup().replace(".", "/"), dep.getName(), dep.getVersion(), |
| 75 | + dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE); |
| 76 | + } |
| 77 | + |
| 78 | + private void downloadDependency(DependencyDownload dt, Path outputDirectory) { |
| 79 | + |
| 80 | + var p = outputDirectory.resolve(dt.filename()); |
| 81 | + var request = HttpRequest.newBuilder().uri(dt.uri()).GET().build(); |
| 82 | + try { |
| 83 | + var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); |
| 84 | + if (response.statusCode() != 200) { |
| 85 | + getLogger().warn("Could not download {}, HTTP response: {}", dt.dependency, response); |
| 86 | + return; |
| 87 | + } |
| 88 | + outputDirectory.toFile().mkdirs(); |
| 89 | + getLogger().debug("Downloading {} into {}", dt, outputDirectory); |
| 90 | + try (var is = response.body(); var fos = new FileOutputStream(p.toFile())) { |
| 91 | + is.transferTo(fos); |
| 92 | + } |
| 93 | + } catch (IOException | InterruptedException e) { |
| 94 | + throw new RuntimeException(e); |
| 95 | + } |
| 96 | + } |
| 97 | + |
| 98 | + /** |
| 99 | + * Creates a download request for a given dependency, classifier, and type. A download request is successfully created if: |
| 100 | + * <ul> |
| 101 | + * <li>the output directory does not exists</li> |
| 102 | + * <li>the file does not exist locally</li> |
| 103 | + * <li>the file exists locally, but is too old (<24hrs) </li> |
| 104 | + * <li>the file exists locally, but is not readable</li> |
| 105 | + * <li>the file is found in at least one Maven repository. MavenLocal is ignored.</li> |
| 106 | + * </ul> |
| 107 | + * |
| 108 | + * @param dep the dependency to download |
| 109 | + * @return an optional DownloadRequest if the artifact should be downloaded, otherwise an empty optional |
| 110 | + */ |
| 111 | + private Optional<DependencyDownload> createDownloadRequest(Dependency dep) { |
| 112 | + if (isLocalFileValid(dep)) { |
| 113 | + getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); |
| 114 | + return Optional.empty(); |
| 115 | + } |
| 116 | + var repos = getProject().getRepositories().stream().toList(); |
| 117 | + return repos.stream() |
| 118 | + .filter(repo -> repo instanceof MavenArtifactRepository) |
| 119 | + .map(repo -> (MavenArtifactRepository) repo) |
| 120 | + .map(repo -> { |
| 121 | + var repoUrl = createArtifactUrl(dep, repo); |
| 122 | + try { |
| 123 | + // we use a HEAD request, because we only want to see whether that module has a `-manifest.json` |
| 124 | + var uri = URI.create(repoUrl); |
| 125 | + var headRequest = HttpRequest.newBuilder() |
| 126 | + .uri(uri) |
| 127 | + .method("HEAD", HttpRequest.BodyPublishers.noBody()) |
| 128 | + .build(); |
| 129 | + var response = httpClient.send(headRequest, HttpResponse.BodyHandlers.discarding()); |
| 130 | + if (response.statusCode() == 200) { |
| 131 | + return new DependencyDownload(dep, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE); |
| 132 | + } |
| 133 | + return null; |
| 134 | + } catch (IOException | InterruptedException | IllegalArgumentException e) { |
| 135 | + return null; |
| 136 | + } |
| 137 | + }) |
| 138 | + .filter(Objects::nonNull) |
| 139 | + .findFirst(); |
| 140 | + } |
| 141 | + |
| 142 | + /** |
| 143 | + * Checks if the manifest for a dependency exists locally. A local file is considered valid if: |
| 144 | + * <ul> |
| 145 | + * <li>The output directory exists</li> |
| 146 | + * <li>The file exists locally and is readable</li> |
| 147 | + * <li>The file is not older than 24 hours</li> |
| 148 | + * </ul> |
| 149 | + * |
| 150 | + * @param dep the dependency to check |
| 151 | + * @return true if the local file is valid, false otherwise |
| 152 | + */ |
| 153 | + private boolean isLocalFileValid(Dependency dep) { |
| 154 | + if (!downloadDirectory.toFile().exists()) return false; |
| 155 | + var filePath = downloadDirectory.resolve(new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); |
| 156 | + var file = filePath.toFile(); |
| 157 | + if (!file.exists() || !file.canRead()) return false; |
| 158 | + |
| 159 | + try { |
| 160 | + var date = Files.getLastModifiedTime(filePath).toInstant(); |
| 161 | + return Duration.between(date, Instant.now()).compareTo(MAX_MANIFEST_AGE) <= 0; |
| 162 | + } catch (IOException e) { |
| 163 | + throw new RuntimeException(e); |
| 164 | + } |
| 165 | + } |
| 166 | + |
| 167 | + private Set<String> getExclusions() { |
| 168 | + return Set.of(); |
| 169 | + } |
| 170 | + |
| 171 | + private record DependencyDownload(Dependency dependency, URI uri, String classifier, String type) { |
| 172 | + @Override |
| 173 | + public String toString() { |
| 174 | + return "{" + |
| 175 | + "dependency=" + dependency + |
| 176 | + ", uri=" + uri + |
| 177 | + '}'; |
| 178 | + } |
| 179 | + |
| 180 | + String filename() { |
| 181 | + return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type); |
| 182 | + } |
| 183 | + } |
| 184 | +} |
0 commit comments