From 8dc8e62965d79be332ad2fc6778df50fcbc10d2f Mon Sep 17 00:00:00 2001 From: Matthias Klumpp Date: Sat, 18 Nov 2023 06:52:20 +0100 Subject: [PATCH] rpmmd: Download packages on-demand --- src/asgen/backends/rpmmd/rpmpkg.d | 45 ++++++++++++++++-- src/asgen/backends/rpmmd/rpmpkgindex.d | 32 ++++++++++--- src/asgen/backends/rpmmd/rpmutils.d | 64 ++++++++++++++++++++++++++ src/asgen/meson.build | 1 + 4 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 src/asgen/backends/rpmmd/rpmutils.d diff --git a/src/asgen/backends/rpmmd/rpmpkg.d b/src/asgen/backends/rpmmd/rpmpkg.d index 4a1c9cd..6c0f23a 100644 --- a/src/asgen/backends/rpmmd/rpmpkg.d +++ b/src/asgen/backends/rpmmd/rpmpkg.d @@ -22,9 +22,14 @@ module asgen.backends.rpmmd.rpmpkg; import std.stdio; import std.string; import std.array : empty; +import std.path : buildNormalizedPath, baseName; +static import std.file; +import asgen.config : Config; import asgen.logging; import asgen.zarchive; +import asgen.downloader : Downloader; +import asgen.utils : isRemote; import asgen.backends.interfaces; final class RPMPackage : Package { @@ -36,6 +41,7 @@ private: string[string] desc; string[string] summ; string pkgFname; + string localPkgFname; string[] contentsL; @@ -81,10 +87,26 @@ public: return desc; } - override - @property string getFilename () const - { - return pkgFname; + override final + @property + string getFilename () + { + if (!localPkgFname.empty) + return localPkgFname; + + if (pkgFname.isRemote) { + synchronized (this) { + auto conf = Config.get(); + auto dl = Downloader.get; + immutable path = buildNormalizedPath(conf.getTmpDir(), format("%s-%s_%s_%s", name, ver, arch, pkgFname.baseName)); + dl.downloadFile(pkgFname, path); + localPkgFname = path; + return localPkgFname; + } + } else { + localPkgFname = pkgFname; + return pkgFname; + } } @property void filename (string fname) @@ -137,5 +159,20 @@ public: override void finish () { + synchronized (this) { + if (archive.isOpen) + archive.close(); + + try { + if (pkgFname.isRemote && std.file.exists(localPkgFname)) { + logDebug("Deleting temporary package file %s", localPkgFname); + localPkgFname = null; + std.file.remove(localPkgFname); + } + } catch (Exception e) { + // we ignore any error + logDebug("Unable to remove temporary package: %s (%s)", localPkgFname, e.msg); + } + } } } diff --git a/src/asgen/backends/rpmmd/rpmpkgindex.d b/src/asgen/backends/rpmmd/rpmpkgindex.d index 31f5b0c..dd1b5c8 100644 --- a/src/asgen/backends/rpmmd/rpmpkgindex.d +++ b/src/asgen/backends/rpmmd/rpmpkgindex.d @@ -20,7 +20,7 @@ module asgen.backends.rpmmd.rpmpkgindex; import std.stdio : writeln; -import std.path : buildPath; +import std.path : buildPath, baseName; import std.array : appender, empty; import std.string : format; import std.algorithm : canFind, endsWith; @@ -29,22 +29,30 @@ import dxml.dom : parseDOM, EntityType; static import std.file; import asgen.logging; +import asgen.config; +import asgen.utils : escapeXml, getTextFileContents, isRemote; + import asgen.backends.interfaces; import asgen.backends.rpmmd.rpmpkg; +import asgen.backends.rpmmd.rpmutils : downloadIfNecessary; final class RPMPackageIndex : PackageIndex { private: string rootDir; Package[][string] pkgCache; + string tmpRootDir; public: this (string dir) { this.rootDir = dir; - if (!std.file.exists(dir)) - throw new Exception("Directory '%s' does not exist.", dir); + if (!dir.isRemote && !std.file.exists(dir)) + throw new Exception("Directory '%s' does not exist.".format(dir)); + + auto conf = Config.get(); + tmpRootDir = buildPath(conf.getTmpDir, dir.baseName); } void release () @@ -82,10 +90,14 @@ public: private RPMPackage[] loadPackages (string suite, string section, string arch) { auto repoRoot = buildPath(rootDir, suite, section, arch, "os"); - auto primaryIndexFiles = appender!(string[]); auto filelistFiles = appender!(string[]); - immutable repoMdIndexContent = cast(string) std.file.read(buildPath(repoRoot, "repodata", "repomd.xml")); + + string repoMdFname; + synchronized (this) + repoMdFname = downloadIfNecessary(buildPath(repoRoot, "repodata", "repomd.xml"), tmpRootDir); + + immutable repoMdIndexContent = cast(string) std.file.read(repoMdFname); // parse index data auto indexDoc = parseDOM(repoMdIndexContent); @@ -115,7 +127,10 @@ public: // parse the primary metadata foreach (ref primaryFile; primaryIndexFiles.data) { - immutable metaFname = buildPath(repoRoot, primaryFile); + string metaFname; + synchronized(this) + metaFname = downloadIfNecessary(buildPath(repoRoot, primaryFile), tmpRootDir); + string data; if (primaryFile.endsWith(".xml")) { data = cast(string) std.file.read(metaFname); @@ -188,7 +203,10 @@ public: // read the filelists foreach (ref filelistFile; filelistFiles.data) { - immutable flistFname = buildPath(repoRoot, filelistFile); + string flistFname; + synchronized (this) + flistFname = downloadIfNecessary(buildPath(repoRoot, filelistFile), tmpRootDir); + string data; if (filelistFile.endsWith(".xml")) { data = cast(string) std.file.read(flistFname); diff --git a/src/asgen/backends/rpmmd/rpmutils.d b/src/asgen/backends/rpmmd/rpmutils.d new file mode 100644 index 0000000..1712f80 --- /dev/null +++ b/src/asgen/backends/rpmmd/rpmutils.d @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2016-2023 Matthias Klumpp + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module asgen.backends.rpmmd.rpmutils; + +import std.string : format; +import std.path : buildPath, baseName; +static import std.file; + +import asgen.logging; +import asgen.downloader : Downloader, DownloadException; +import asgen.utils : isRemote; + +/** + * If URL is remote, download it, otherwise use it verbatim. + * + * Returns: Path to the file, which is guaranteed to exist. + * + * Params: + * url = First part of the address, i.e. + * "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/" + * destPrefix = If the file is remote, the directory to save it under, + * which is created if necessary. + */ +immutable(string) downloadIfNecessary (const string url, + const string destLocation, + Downloader downloader = null) +{ + import std.path : buildPath; + + if (downloader is null) + downloader = Downloader.get; + + if (url.isRemote) { + immutable destFileName = buildPath(destLocation, url.baseName); + try { + downloader.downloadFile(url, destFileName); + return destFileName; + } catch (DownloadException e) { + logDebug("Unable to download: %s", e.msg); + } + } else { + if (std.file.exists(url)) + return url; + } + + throw new Exception("Could not obtain any file %s".format(url)); +} diff --git a/src/asgen/meson.build b/src/asgen/meson.build index 9d7136c..678681d 100644 --- a/src/asgen/meson.build +++ b/src/asgen/meson.build @@ -75,6 +75,7 @@ if get_option('rpmmd') 'backends/rpmmd/package.d', 'backends/rpmmd/rpmpkg.d', 'backends/rpmmd/rpmpkgindex.d', + 'backends/rpmmd/rpmutils.d', ] endif