rpmmd: Download packages on-demand

src/asgen/backends/rpmmd/rpmpkg.d

@@ -22,9 +22,14 @@ module asgen.backends.rpmmd.rpmpkg;
 import std.stdio;
 import std.string;
 import std.array : empty;
+import std.path : buildNormalizedPath, baseName;
+static import std.file;
 
+import asgen.config : Config;
 import asgen.logging;
 import asgen.zarchive;
+import asgen.downloader : Downloader;
+import asgen.utils : isRemote;
 import asgen.backends.interfaces;
 
 final class RPMPackage : Package {
@@ -36,6 +41,7 @@ private:
     string[string] desc;
     string[string] summ;
     string pkgFname;
+    string localPkgFname;
 
     string[] contentsL;
 
@@ -81,10 +87,26 @@ public:
         return desc;
     }
 
-    override
-    @property string getFilename () const
-    {
-        return pkgFname;
+    override final
+    @property
+    string getFilename ()
+    {
+        if (!localPkgFname.empty)
+            return localPkgFname;
+
+        if (pkgFname.isRemote) {
+            synchronized (this) {
+                auto conf = Config.get();
+                auto dl = Downloader.get;
+                immutable path = buildNormalizedPath(conf.getTmpDir(), format("%s-%s_%s_%s", name, ver, arch, pkgFname.baseName));
+                dl.downloadFile(pkgFname, path);
+                localPkgFname = path;
+                return localPkgFname;
+            }
+        } else {
+            localPkgFname = pkgFname;
+            return pkgFname;
+        }
     }
 
     @property void filename (string fname)
@@ -137,5 +159,20 @@ public:
     override
     void finish ()
     {
+        synchronized (this) {
+            if (archive.isOpen)
+                archive.close();
+
+            try {
+                if (pkgFname.isRemote && std.file.exists(localPkgFname)) {
+                    logDebug("Deleting temporary package file %s", localPkgFname);
+                    localPkgFname = null;
+                    std.file.remove(localPkgFname);
+                }
+            } catch (Exception e) {
+                // we ignore any error
+                logDebug("Unable to remove temporary package: %s (%s)", localPkgFname, e.msg);
+            }
+        }
     }
 }

src/asgen/backends/rpmmd/rpmpkgindex.d

@@ -20,7 +20,7 @@
 module asgen.backends.rpmmd.rpmpkgindex;
 
 import std.stdio : writeln;
-import std.path : buildPath;
+import std.path : buildPath, baseName;
 import std.array : appender, empty;
 import std.string : format;
 import std.algorithm : canFind, endsWith;
@@ -29,22 +29,30 @@ import dxml.dom : parseDOM, EntityType;
 static import std.file;
 
 import asgen.logging;
+import asgen.config;
+import asgen.utils : escapeXml, getTextFileContents, isRemote;
+
 import asgen.backends.interfaces;
 import asgen.backends.rpmmd.rpmpkg;
+import asgen.backends.rpmmd.rpmutils : downloadIfNecessary;
 
 final class RPMPackageIndex : PackageIndex {
 
 private:
     string rootDir;
     Package[][string] pkgCache;
+    string tmpRootDir;
 
 public:
 
     this (string dir)
     {
         this.rootDir = dir;
-        if (!std.file.exists(dir))
-            throw new Exception("Directory '%s' does not exist.", dir);
+        if (!dir.isRemote && !std.file.exists(dir))
+            throw new Exception("Directory '%s' does not exist.".format(dir));
+
+        auto conf = Config.get();
+        tmpRootDir = buildPath(conf.getTmpDir, dir.baseName);
     }
 
     void release ()
@@ -82,10 +90,14 @@ public:
     private RPMPackage[] loadPackages (string suite, string section, string arch)
     {
         auto repoRoot = buildPath(rootDir, suite, section, arch, "os");
-
         auto primaryIndexFiles = appender!(string[]);
         auto filelistFiles = appender!(string[]);
-        immutable repoMdIndexContent = cast(string) std.file.read(buildPath(repoRoot, "repodata", "repomd.xml"));
+
+        string repoMdFname;
+        synchronized (this)
+            repoMdFname = downloadIfNecessary(buildPath(repoRoot, "repodata", "repomd.xml"), tmpRootDir);
+
+        immutable repoMdIndexContent = cast(string) std.file.read(repoMdFname);
 
         // parse index data
         auto indexDoc = parseDOM(repoMdIndexContent);
@@ -115,7 +127,10 @@ public:
 
         // parse the primary metadata
         foreach (ref primaryFile; primaryIndexFiles.data) {
-            immutable metaFname = buildPath(repoRoot, primaryFile);
+            string metaFname;
+            synchronized(this)
+                metaFname = downloadIfNecessary(buildPath(repoRoot, primaryFile), tmpRootDir);
+
             string data;
             if (primaryFile.endsWith(".xml")) {
                 data = cast(string) std.file.read(metaFname);
@@ -188,7 +203,10 @@ public:
 
         // read the filelists
         foreach (ref filelistFile; filelistFiles.data) {
-            immutable flistFname = buildPath(repoRoot, filelistFile);
+            string flistFname;
+            synchronized (this)
+                flistFname = downloadIfNecessary(buildPath(repoRoot, filelistFile), tmpRootDir);
+
             string data;
             if (filelistFile.endsWith(".xml")) {
                 data = cast(string) std.file.read(flistFname);

src/asgen/backends/rpmmd/rpmutils.d

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016-2023 Matthias Klumpp <[email protected]>
+ *
+ * Licensed under the GNU Lesser General Public License Version 3
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the license, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this software.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+module asgen.backends.rpmmd.rpmutils;
+
+import std.string : format;
+import std.path : buildPath, baseName;
+static import std.file;
+
+import asgen.logging;
+import asgen.downloader : Downloader, DownloadException;
+import asgen.utils : isRemote;
+
+/**
+ * If URL is remote, download it, otherwise use it verbatim.
+ *
+ * Returns: Path to the file, which is guaranteed to exist.
+ *
+ * Params:
+ *      url = First part of the address, i.e.
+ *               "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/"
+ *      destPrefix = If the file is remote, the directory to save it under,
+ *                   which is created if necessary.
+ */
+immutable(string) downloadIfNecessary (const string url,
+        const string destLocation,
+        Downloader downloader = null)
+{
+    import std.path : buildPath;
+
+    if (downloader is null)
+        downloader = Downloader.get;
+
+    if (url.isRemote) {
+        immutable destFileName = buildPath(destLocation, url.baseName);
+        try {
+            downloader.downloadFile(url, destFileName);
+            return destFileName;
+        } catch (DownloadException e) {
+            logDebug("Unable to download: %s", e.msg);
+        }
+    } else {
+        if (std.file.exists(url))
+            return url;
+    }
+
+    throw new Exception("Could not obtain any file %s".format(url));
+}

src/asgen/meson.build

@@ -75,6 +75,7 @@ if get_option('rpmmd')
         'backends/rpmmd/package.d',
         'backends/rpmmd/rpmpkg.d',
         'backends/rpmmd/rpmpkgindex.d',
+        'backends/rpmmd/rpmutils.d',
     ]
 endif