diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index c7774f120c..0250a16ccc 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -267,35 +267,49 @@ jobs: # access to quay.io/bioconda. push: name: push images - if: (github.ref == 'refs/heads/master') && (needs.detect-existing.outputs.DO_BUILD == 'true') + if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} + needs: [ test, build-images, detect-existing ] runs-on: ubuntu-24.04 - needs: [ test ] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + # Download tarballs created in the previous job + - name: Download images as artifacts + uses: actions/download-artifact@v4 + if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} + with: + name: image-artifacts + path: image-artifacts + + # Load those tarballs as images into podman. + - name: Load image artifacts into podman + if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} + run: | + for image in image-artifacts/*.tar; do + podman load -i $image + done + - name: push manifests to quay.io run: | - # quay.io login - echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin + set -x + if [ ${{ github.head_ref }} == "master" ]; then - # Source env vars and functions to be used throughout building. - source images/image_config.sh + echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin - # Compose a multi-arch manifest (json file); push it and its images to quay.io - build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" - build_and_push_manifest ${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" - build_and_push_manifest ${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" - build_and_push_manifest ${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" - ONLY_AMD64=true build_and_push_manifest ${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" - - # Now that they are pushed, we can make tags pointing to them. Note - # that the "latest" tag will not be applied unil release time (via - # release-please workflow). - podman tag "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" - podman tag "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" - podman tag "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}::master" - podman tag "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" - podman tag "quay.io/bioconda/${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" "quay.io/bioconda/${BOT_IMAGE_NAME}:master" + # Source env vars and functions to be used throughout building. + source images/image_config.sh + + # Compose a multi-arch manifest (json file); push it and its images + # to quay.io with the "master" tag. + # Issues uploading? See the notes in image_config.sh on repository creation. + build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" + build_and_push_manifest ${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG} "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" + build_and_push_manifest ${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master" + build_and_push_manifest ${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" + ONLY_AMD64=true build_and_push_manifest ${BOT_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG} "quay.io/bioconda/${BOT_IMAGE_NAME}:master" + + + fi diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 82f120048b..ba50765cd4 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -32,16 +32,17 @@ jobs: #tag=${{ github.event.release && github.event.release.tag_name || github.sha }} tag=${{ needs.release_please.outputs.tag_name }} printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT - + - name: tag images with latest run: | echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin source images/image_config.sh - podman tag "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest" - podman tag "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:latest" - podman tag "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master" "quay.io/bioconda${BUILD_ENV_IMAGE_NAME}:latest" - podman tag "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:latest" - podman tag "quay.io/bioconda/${BOT_IMAGE_NAME}:master" "quay.io/bioconda/${BOT_IMAGE_NAME}:latest" + # build_and_push_manifest + build_and_push_manifest "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest" "" "false" + build_and_push_manifest "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:master" "quay.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:latest" "" "false" + build_and_push_manifest "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${BUILD_ENV_IMAGE_NAME}:latest" "" "false" + build_and_push_manifest "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:master" "quay.io/bioconda/${CREATE_ENV_IMAGE_NAME}:latest" "" "false" + ONLY_AMD64=true build_and_push_manifest "quay.io/bioconda/${BOT_IMAGE_NAME}:master" "quay.io/bioconda/${BOT_IMAGE_NAME}:latest" "" "false" diff --git a/images/image_config.sh b/images/image_config.sh index 6c4be72cfd..f7b18aa670 100644 --- a/images/image_config.sh +++ b/images/image_config.sh @@ -2,17 +2,24 @@ #----------------------------VERSIONS------------------------------------------- # Configures various versions to be used throughout infrastructure. +# +# NOTE: If you change a container name, you need to make sure you create the +# respective repository on quay.io/bioconda ("+Create New Repository" in top +# right) AND in the new repository settings you need to give the +# bioconda+bioconda_utils_bot write access to the repository. ARCHS="amd64 arm64" DEBIAN_VERSION="12.5" BUSYBOX_VERSION="1.36.1" -BASE_DEBIAN_IMAGE_NAME="tmp-base-debian" -BASE_BUSYBOX_IMAGE_NAME="tmp-base-busybox" +BASE_DEBIAN_IMAGE_NAME="tmp-debian" +BASE_BUSYBOX_IMAGE_NAME="tmp-busybox" BUILD_ENV_IMAGE_NAME="tmp-build-env" CREATE_ENV_IMAGE_NAME="tmp-create-env" BOT_IMAGE_NAME="tmp-bot" BASE_TAG="0.2" BASE_IMAGE_CONDAFORGE_AMD64="quay.io/condaforge/linux-anvil-x86_64:cos7" BASE_IMAGE_CONDAFORGE_ARM64="quay.io/condaforge/linux-anvil-aarch64:cos7" + +# May be set within loops sourcing this script, in which case respect it CURRENT_ARCH=${CURRENT_ARCH:-""} # Inspect this repo to get the currently-checked-out version, which matches @@ -67,19 +74,28 @@ function tag_exists() { } function build_and_push_manifest() { - # Creates a local manifest, adds containers for multiple archs, and pushes to - # a registry. + # Creates a local manifest, pull containers for multiple archs, adds them to + # the manifest, and pushes the manifest & images to a registry. + # + # build_and_push_manifest + # + # (last two args are optional) # - # Typical usage: - # build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} docker://localhost:5000 ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} "--tls-verify=false" + # For pushing to a local registry: # - # or + # build_and_push_manifest image:tag docker://localhost:5000/image:tag "--tls-verify=false" # - # build_and_push_manifest ${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG} quay.io/bioconda/${BASE_BUSYBOX_IMAGE_NAME}:latest + # For pushing the manifest from local images to quay.io: # + # build_and_push_manifest image:tag quay.io/bioconda/image:master + # + # For re-tagging a manifest already on quay.io (with its component images): + # + # build_and_push_manifest quay.io/bioconda/image:master quay.io/bioconda/image:latest "" "false" local source=$1 local dest=$2 local additional_args=${3:-""} + local use_arch_suffix=${4:-"true"} local manifest_name="local_${source}" @@ -95,7 +111,17 @@ function build_and_push_manifest() { # skip non-amd64 if configured [ "${ONLY_AMD64:-false}" == "true" -a "${arch}" != "amd64" ] && continue - imgid=$(buildah pull --arch=$arch "${source}-${arch}") + # When first creating a manifest, we have local images that have + # arch-specific suffixes so use the default, "true". + # + # However, when re-tagging a manifest already on quay.io, the arch-specific + # images are inside the manifest and do not have arch-specific suffixes, so + # use "false". + if [ $use_arch_suffix == "true" ]; then + imgid=$(buildah pull --arch=$arch "${source}-${arch}") + else + imgid=$(buildah pull --arch=$arch "${source}") + fi buildah manifest add "${manifest_name}" "${imgid}" done @@ -104,7 +130,6 @@ function build_and_push_manifest() { podman manifest push --all $additional_args "${manifest_name}" "${dest}" } - function env_var_inventory () { # There are a lot of environment variables used here; this function describes # them and reports their values at call time.