RFC download keyring debs files too

Author: igorpecovnik

.github/workflows/generate-keyring-data.yaml

@@ -1,4 +1,5 @@
 name: "Generate list of latest keyrings for Debian & Ubuntu"
+
 on:
   workflow_dispatch:
   repository_dispatch:
@@ -13,41 +14,145 @@ jobs:
     runs-on: ubuntu-24.04
     name: "Generate Keyring Data"
     steps:
-
       - name: Checkout repository
         uses: actions/checkout@v5
         with:
           fetch-depth: 0
           path: armbian.github.io
 
-      - name: "Generate keyring files"
+      - name: "Find and download latest keyrings"
+        shell: bash
         run: |
-          NEWEST_SUITE=$(curl --max-time 30 --compressed -fLs https://changelogs.ubuntu.com/meta-release | grep '^Dist:'| tail -n 1 | awk '{print $NF}')
-          # NOTE: this service on PUC returns a long list of ISO-3166-2 country code prefixed archive.ubuntu.com mirrors [among others]
-          # We remove that prefix as it's better to use the rotation. the number in the regex is b/c nz has nz and nz2
-          # example URL returned: http://nz2.archive.ubuntu.com/ubuntu/pool/main/u/ubuntu-keyring/ubuntu-keyring_2023.11.28.1_all.deb
-          PKG_URL=$(curl --max-time 30 --compressed -fLs "https://packages.ubuntu.com/${NEWEST_SUITE}/all/ubuntu-keyring/download" | \
-            grep -oP 'https?://\S+archive.ubuntu.com/ubuntu/pool/main/u/\S+\.deb' | tail -n 1 | sed -E 's#://[a-z][a-z][0-9]?\.#://#')
-          [[ -z "${PKG_URL}" ]] && (echo "fetch_newest_keyring failed - unable to find newest ubuntu-keyring package"; exit 1)
-          echo $PKG_URL > latest-ubuntu-keyring.txt
-
-          for p in debian-archive-keyring debian-ports-archive-keyring; do
-            PKG_URL=$(curl --max-time 30 --compressed -fLs "https://packages.debian.org/sid/all/${p}/download" | \
-              grep -oP "https?://(deb|ftp)\.debian\.org/debian/pool/main/d/${p}/${p}_\S+\.deb")
-            [[ -z "${PKG_URL}" ]] && (echo "fetch_newest_keyring failed - unable to find newest $p package"; exit 1)
-            echo $PKG_URL > latest-$p.txt
+          set -euo pipefail
+
+          retry_curl() {
+            # Usage: retry_curl <url> [output]
+            local url="$1"
+            local out="${2:-}"
+            if [[ -n "$out" ]]; then
+              curl --max-time 60 --retry 3 --retry-all-errors --compressed -fL "$url" -o "$out"
+            else
+              curl --max-time 60 --retry 3 --retry-all-errors --compressed -fL "$url"
+            fi
+          }
+
+          workdir="$(mktemp -d)"
+          echo "Workdir: $workdir"
+
+          # --- Ubuntu latest suite and keyring URL ---
+          # Use Ubuntu meta-release to determine the newest listed suite.
+          NEWEST_SUITE=$(retry_curl "https://changelogs.ubuntu.com/meta-release" \
+            | grep '^Dist:' | awk '{print $NF}' | tail -n 1)
+
+          if [[ -z "${NEWEST_SUITE:-}" ]]; then
+            echo "ERROR: Unable to detect Ubuntu newest suite" >&2
+            exit 1
+          fi
+
+          echo "Newest Ubuntu suite: $NEWEST_SUITE"
+
+          # Grab an archive.ubuntu.com link to ubuntu-keyring .deb, prefer rotation (drop cc/ccN prefix).
+          UB_PAGE_URL="https://packages.ubuntu.com/${NEWEST_SUITE}/all/ubuntu-keyring/download"
+          UB_PKG_URL=$(retry_curl "$UB_PAGE_URL" \
+            | grep -oP 'https?://\S+archive\.ubuntu\.com/ubuntu/pool/main/u/\S+\.deb' \
+            | tail -n 1 \
+            | sed -E 's#://[a-z][a-z][0-9]?\.#://#')
+
+          if [[ -z "${UB_PKG_URL:-}" ]]; then
+            echo "ERROR: Unable to find ubuntu-keyring package URL from $UB_PAGE_URL" >&2
+            exit 1
+          fi
+
+          echo "Ubuntu keyring URL: $UB_PKG_URL"
+
+          # --- Debian keyrings (sid pages list the latest available versions) ---
+          declare -a DEB_PKGS=(debian-archive-keyring debian-ports-archive-keyring)
+          declare -A DEB_URLS=()
+
+          for p in "${DEB_PKGS[@]}"; do
+            DEB_PAGE_URL="https://packages.debian.org/sid/all/${p}/download"
+            # Prefer deb.debian.org or ftp.debian.org
+            url=$(retry_curl "$DEB_PAGE_URL" \
+              | grep -oP 'https?://(deb|ftp)\.debian\.org/debian/pool/main/d/[^/]+/[^"]+\.deb' \
+              | head -n 1 || true)
+            if [[ -z "${url:-}" ]]; then
+              echo "ERROR: Unable to find ${p} package URL from $DEB_PAGE_URL" >&2
+              exit 1
+            fi
+            DEB_URLS["$p"]="$url"
+            echo "Debian $p URL: ${DEB_URLS[$p]}"
+          done
+
+          # --- Download all files to the temp workdir ---
+          declare -a DOWNLOADED=()
+
+          fname_ubuntu="$(basename "$UB_PKG_URL")"
+          retry_curl "$UB_PKG_URL" "$workdir/$fname_ubuntu"
+          DOWNLOADED+=("$workdir/$fname_ubuntu")
+
+          for p in "${DEB_PKGS[@]}"; do
+            f="$(basename "${DEB_URLS[$p]}")"
+            retry_curl "${DEB_URLS[$p]}" "$workdir/$f"
+            DOWNLOADED+=("$workdir/$f")
+          done
+
+          # --- Stage into repo folder armbian.github.io/keyrings ---
+          pushd armbian.github.io >/dev/null
+
+          # Work on the 'data' branch as before (create if missing locally)
+          if ! git rev-parse --verify data >/dev/null 2>&1; then
+            git fetch origin data || true
+          fi
+          git checkout data
+
+          mkdir -p keyrings
+
+          # Move files in (overwrite existing versions)
+          for f in "${DOWNLOADED[@]}"; do
+            base="$(basename "$f")"
+            mv -f "$f" "keyrings/$base"
           done
+
+          # --- Create/update symlinks ---
+          # Per-package latest symlinks
+          # Resolve the expected basenames we just saved to infer symlink targets.
+          ub_target="$fname_ubuntu"
+          ln -sfn "$ub_target" "keyrings/latest-ubuntu-keyring.deb"
+
+          # Find the files we downloaded for Debian packages and symlink them
+          for p in "${DEB_PKGS[@]}"; do
+            # match first file starting with package name
+            cand="$(ls -1 keyrings/${p}_*.deb 2>/dev/null | sort | tail -n 1 || true)"
+            if [[ -n "$cand" ]]; then
+              ln -sfn "$(basename "$cand")" "keyrings/latest-${p}.deb"
+            fi
+          done
+
+          # Generic 'latest' -> most recently modified of the just-downloaded files in keyrings
+          latest_target="$(ls -1t keyrings/*.deb | head -n 1)"
+          if [[ -n "$latest_target" ]]; then
+            ln -sfn "$(basename "$latest_target")" "keyrings/latest"
+          fi
+
+          popd >/dev/null
+
       - name: Commit changes if any
+        shell: bash
         run: |
+          set -euo pipefail
           cd armbian.github.io
           git checkout data
-          mkdir -p data/
-          mv ${{ github.workspace }}/latest-*keyring*.txt data/
+
           git config --global user.name "github-actions"
           git config --global user.email "[email protected]"
-          git add data/.
-          git diff --cached --quiet || git commit -m "Update keyring data files"
-          git push
+          git add keyrings/
+
+          if ! git diff --cached --quiet; then
+            git commit -m "Update keyrings: download latest .deb files and symlinks"
+            git push
+          else
+            echo "No changes to commit."
+          fi
 
       - name: "Run Bigin update action"
         uses: peter-evans/repository-dispatch@v4