test: [WIP] Containers overhaul upload test

.github/workflows/build-images.yml

@@ -267,35 +267,49 @@ jobs:
   # access to quay.io/bioconda.
   push:
     name: push images
-    if: (github.ref == 'refs/heads/master') && (needs.detect-existing.outputs.DO_BUILD == 'true')
+    if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }}
+    needs: [ test, build-images, detect-existing ]
     runs-on: ubuntu-24.04
-    needs: [ test ]
     steps:
 
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
+    # Download tarballs created in the previous job
+    - name: Download images as artifacts
+      uses: actions/download-artifact@v4
+      if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }}
+      with:
+        name: image-artifacts
+        path: image-artifacts
+
+    # Load those tarballs as images into podman.
+    - name: Load image artifacts into podman
+      if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }}
+      run: |
+          for image in image-artifacts/*.tar; do
+            podman load -i $image
+          done
+
     - name: push manifests to quay.io
       run: |
-        # quay.io login
-        echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
+        set -x
+        if [ ${{ github.head_ref }} == "master" ]; then
 
-        # Source env vars and functions to be used throughout building.
-        source images/image_config.sh
+          echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
 
-        # Compose a multi-arch manifest (json file); push it and its images to quay.io
-        build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}"
-        build_and_push_manifest ${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}"
-        build_and_push_manifest ${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
-        build_and_push_manifest ${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
-        ONLY_AMD64=true build_and_push_manifest ${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
-
-        # Now that they are pushed, we can make tags pointing to them. Note
-        # that the "latest" tag will not be applied unil release time (via
-        # release-please workflow).
-        podman tag "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master"
-        podman tag "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master"
-        podman tag "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}::master"
-        podman tag "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master"
-        podman tag "quay.io/bioconda/${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda/${BOT_IMAGE_NAME}:master"
+          # Source env vars and functions to be used throughout building.
+          source images/image_config.sh
+
+          # Compose a multi-arch manifest (json file); push it and its images
+          # to quay.io with the "master" tag.
+          # Issues uploading? See the notes in image_config.sh on repository creation.
+          build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master"
+          build_and_push_manifest ${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master"
+          build_and_push_manifest ${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master"
+          build_and_push_manifest ${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master"
+          ONLY_AMD64=true build_and_push_manifest ${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BOT_IMAGE_NAME}:master"
+
+
+        fi

.github/workflows/release-please.yml

@@ -32,16 +32,17 @@ jobs:
           #tag=${{ github.event.release && github.event.release.tag_name || github.sha }}
           tag=${{ needs.release_please.outputs.tag_name }}
           printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT
-      
+
       - name: tag images with latest
         run: |
 
           echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
 
           source images/image_config.sh
-          podman tag "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest"
-          podman tag "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:latest"
-          podman tag "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master" "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}:latest"
-          podman tag "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:latest"
-          podman tag "quay.io/bioconda/${BOT_IMAGE_NAME}:master" "quay.io/bioconda/${BOT_IMAGE_NAME}:latest"
 
+          # build_and_push_manifest <source> <dest> <additional args> <use arch-specific suffix>
+          build_and_push_manifest "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest" "" "false"
+          build_and_push_manifest "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:latest" "" "false"
+          build_and_push_manifest "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:latest" "" "false"
+          build_and_push_manifest "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:latest" "" "false"
+          ONLY_AMD64=true build_and_push_manifest "quay.io/bioconda/${BOT_IMAGE_NAME}:master" "quay.io/bioconda/${BOT_IMAGE_NAME}:latest" "" "false"

images/image_config.sh

@@ -2,17 +2,24 @@
 
 #----------------------------VERSIONS-------------------------------------------
 # Configures various versions to be used throughout infrastructure.
+#
+# NOTE: If you change a container name, you need to make sure you create the
+# respective repository on quay.io/bioconda ("+Create New Repository" in top
+# right) AND in the new repository settings you need to give the
+# bioconda+bioconda_utils_bot write access to the repository.
 ARCHS="amd64 arm64"
 DEBIAN_VERSION="12.5"
 BUSYBOX_VERSION="1.36.1"
-BASE_DEBIAN_IMAGE_NAME="tmp-base-debian"
-BASE_BUSYBOX_IMAGE_NAME="tmp-base-busybox"
+BASE_DEBIAN_IMAGE_NAME="tmp-debian"
+BASE_BUSYBOX_IMAGE_NAME="tmp-busybox"
 BUILD_ENV_IMAGE_NAME="tmp-build-env"
 CREATE_ENV_IMAGE_NAME="tmp-create-env"
 BOT_IMAGE_NAME="tmp-bot"
 BASE_TAG="0.2"
 BASE_IMAGE_CONDAFORGE_AMD64="quay.io/condaforge/linux-anvil-x86_64:cos7"
 BASE_IMAGE_CONDAFORGE_ARM64="quay.io/condaforge/linux-anvil-aarch64:cos7"
+
+# May be set within loops sourcing this script, in which case respect it
 CURRENT_ARCH=${CURRENT_ARCH:-""}
 
 # Inspect this repo to get the currently-checked-out version, which matches
@@ -67,19 +74,28 @@ function tag_exists() {
 }
 
 function build_and_push_manifest() {
-  # Creates a local manifest, adds containers for multiple archs, and pushes to
-  # a registry.
+  # Creates a local manifest, pull containers for multiple archs, adds them to
+  # the manifest, and pushes the manifest & images to a registry.
+  #
+  # build_and_push_manifest <source> <dest> <podman manifest push args> <use_arch_suffix>
+  #
+  # (last two args are optional)
   #
-  # Typical usage:
-  #   build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} docker://localhost:5000 ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "--tls-verify=false"
+  # For pushing to a local registry:
   #
-  # or
+  #   build_and_push_manifest image:tag docker://localhost:5000/image:tag "--tls-verify=false"
   #
-  #   build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest
+  # For pushing the manifest from local images to quay.io:
   #
+  #   build_and_push_manifest image:tag quay.io/bioconda/image:master
+  #
+  # For re-tagging a manifest already on quay.io (with its component images):
+  #
+  #   build_and_push_manifest quay.io/bioconda/image:master quay.io/bioconda/image:latest "" "false"
   local source=$1
   local dest=$2
   local additional_args=${3:-""}
+  local use_arch_suffix=${4:-"true"}
 
   local manifest_name="local_${source}"
 
@@ -95,7 +111,17 @@ function build_and_push_manifest() {
     # skip non-amd64 if configured
     [ "${ONLY_AMD64:-false}" == "true" -a "${arch}" != "amd64" ] && continue
 
-    imgid=$(buildah pull --arch=$arch "${source}-${arch}")
+    # When first creating a manifest, we have local images that have
+    # arch-specific suffixes so use the default, "true".
+    #
+    # However, when re-tagging a manifest already on quay.io, the arch-specific
+    # images are inside the manifest and do not have arch-specific suffixes, so
+    # use "false".
+    if [ $use_arch_suffix == "true" ]; then 
+      imgid=$(buildah pull --arch=$arch "${source}-${arch}")
+    else
+      imgid=$(buildah pull --arch=$arch "${source}")
+    fi
 
     buildah manifest add "${manifest_name}" "${imgid}"
   done
@@ -104,7 +130,6 @@ function build_and_push_manifest() {
   podman manifest push --all $additional_args "${manifest_name}" "${dest}"
 }
 
-
 function env_var_inventory () {
   # There are a lot of environment variables used here; this function describes
   # them and reports their values at call time.