From 860e48dd213af6dd461483972c589714f81cddd2 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 20 Oct 2024 14:03:28 -0400 Subject: [PATCH 1/4] chore: update gh action --- .github/workflows/docker-build.yaml | 541 ++++++++++++++-------------- 1 file changed, 271 insertions(+), 270 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index b38634e1..53fa09ab 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -1,277 +1,278 @@ name: Create and publish Docker images with specific build args on: - workflow_dispatch: - push: - branches: - - main - - dev + workflow_dispatch: + push: + branches: + - main + - dev + - tweaks env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - FULL_IMAGE_NAME: ghcr.io/${{ github.repository }} + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + FULL_IMAGE_NAME: ghcr.io/${{ github.repository }} jobs: - build-main-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - - - name: Extract metadata for Docker cache - id: cache-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - flavor: | - prefix=cache-${{ matrix.platform }}- - - - name: Build Docker image (latest) - uses: docker/build-push-action@v5 - id: build - with: - context: . - push: true - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} - cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max - build-args: | - BUILD_HASH=${{ github.sha }} - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-main-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - build-cuda-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (cuda tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - suffix=-cuda,onlatest=true - - - name: Extract metadata for Docker cache - id: cache-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - flavor: | - prefix=cache-cuda-${{ matrix.platform }}- - - - name: Build Docker image (cuda) - uses: docker/build-push-action@v5 - id: build - with: - context: . - push: true - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} - cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max - build-args: | - BUILD_HASH=${{ github.sha }} - USE_CUDA=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-cuda-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge-main-images: - runs-on: ubuntu-latest - needs: [build-main-image] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: digests-main-* - path: /tmp/digests - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} - - merge-cuda-images: - runs-on: ubuntu-latest - needs: [build-cuda-image] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: digests-cuda-* - path: /tmp/digests - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - suffix=-cuda,onlatest=true - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + build-main-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + + - name: Extract metadata for Docker cache + id: cache-meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + flavor: | + prefix=cache-${{ matrix.platform }}- + + - name: Build Docker image (latest) + uses: docker/build-push-action@v5 + id: build + with: + context: . + push: true + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + build-args: | + BUILD_HASH=${{ github.sha }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-main-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + build-cuda-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (cuda tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-cuda,onlatest=true + + - name: Extract metadata for Docker cache + id: cache-meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + flavor: | + prefix=cache-cuda-${{ matrix.platform }}- + + - name: Build Docker image (cuda) + uses: docker/build-push-action@v5 + id: build + with: + context: . + push: true + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + build-args: | + BUILD_HASH=${{ github.sha }} + USE_CUDA=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-cuda-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-main-images: + runs-on: ubuntu-latest + needs: [build-main-image] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-main-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + + merge-cuda-images: + runs-on: ubuntu-latest + needs: [build-cuda-image] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-cuda-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-cuda,onlatest=true + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} From 865bc273d923e8da76a087243950fe8a3a192792 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 21 Oct 2024 10:11:36 -0400 Subject: [PATCH 2/4] ci: refactor workflows --- .github/workflows/build-docker-image.yaml | 120 +++++++++ .github/workflows/docker-build.yaml | 280 ++------------------- .github/workflows/merge-docker-images.yaml | 71 ++++++ 3 files changed, 215 insertions(+), 256 deletions(-) create mode 100644 .github/workflows/build-docker-image.yaml create mode 100644 .github/workflows/merge-docker-images.yaml diff --git a/.github/workflows/build-docker-image.yaml b/.github/workflows/build-docker-image.yaml new file mode 100644 index 00000000..7f2bd6e3 --- /dev/null +++ b/.github/workflows/build-docker-image.yaml @@ -0,0 +1,120 @@ +name: Build Docker Image + +on: + workflow_call: + inputs: + build_args: + required: false + default: "" + type: string + cache_id: + required: true + type: string + extract_flavor: + required: false + default: "" + type: string + image_name: + required: true + type: string + image_tag: + required: false + default: "" + type: string + registry: + required: false + default: ghcr.io + type: string + +env: + FULL_IMAGE_NAME: ${{ inputs.registry }}/${{ inputs.image_name }} + +jobs: + build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + ${{ inputs.image_tag }} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + ${{ inputs.extract_flavor }} + + - name: Extract metadata for Docker cache + id: cache-meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + flavor: | + prefix=cache-${{ inputs.cache_id }}-${{ matrix.platform }}- + + - name: Build Docker image + uses: docker/build-push-action@v5 + id: build + with: + context: . + push: true + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + build-args: | + BUILD_HASH=${{ github.sha }} + ${{ inputs.build_args }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ inputs.cache_id }}-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 53fa09ab..f376f148 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -6,273 +6,41 @@ on: branches: - main - dev - - tweaks env: - REGISTRY: ghcr.io + CUDA_IMAGE_TAG: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + CUDA_IMAGE_FLAVOR: suffix=-cuda,onlatest=true IMAGE_NAME: ${{ github.repository }} - FULL_IMAGE_NAME: ghcr.io/${{ github.repository }} jobs: build-main-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - - - name: Extract metadata for Docker cache - id: cache-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - flavor: | - prefix=cache-${{ matrix.platform }}- - - - name: Build Docker image (latest) - uses: docker/build-push-action@v5 - id: build - with: - context: . - push: true - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} - cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max - build-args: | - BUILD_HASH=${{ github.sha }} - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-main-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 + uses: ./.github/workflows/build-docker-image.yaml + with: + image_name: ${{ env.IMAGE_NAME }} + cache_id: main build-cuda-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (cuda tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - suffix=-cuda,onlatest=true - - - name: Extract metadata for Docker cache - id: cache-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - flavor: | - prefix=cache-cuda-${{ matrix.platform }}- - - - name: Build Docker image (cuda) - uses: docker/build-push-action@v5 - id: build - with: - context: . - push: true - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} - cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max - build-args: | - BUILD_HASH=${{ github.sha }} - USE_CUDA=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-cuda-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 + uses: ./.github/workflows/build-docker-image.yaml + with: + image_name: ${{ env.IMAGE_NAME }} + cache_id: cuda + image_tag: ${{ env.CUDA_IMAGE_TAG }} + extract_flavor: ${{ env.CUDA_IMAGE_FLAVOR }} + build_args: | + USE_CUDA=true merge-main-images: - runs-on: ubuntu-latest + uses: ./.github/workflows/merge-docker-images.yaml needs: [build-main-image] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: digests-main-* - path: /tmp/digests - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + with: + image_name: ${{ env.IMAGE_NAME }} + cache_id: main merge-cuda-images: - runs-on: ubuntu-latest + uses: ./.github/workflows/merge-docker-images.yaml needs: [build-cuda-image] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: digests-cuda-* - path: /tmp/digests - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for Docker images (default latest tag) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.FULL_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=ref,event=tag - type=sha,prefix=git- - type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda - flavor: | - latest=${{ github.ref == 'refs/heads/main' }} - suffix=-cuda,onlatest=true - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + with: + image_name: ${{ env.IMAGE_NAME }} + cache_id: cuda + extract_flavor: ${{ env.CUDA_IMAGE_FLAVOR }} + extract_tags: ${{ env.CUDA_IMAGE_TAG }} diff --git a/.github/workflows/merge-docker-images.yaml b/.github/workflows/merge-docker-images.yaml new file mode 100644 index 00000000..512f42cf --- /dev/null +++ b/.github/workflows/merge-docker-images.yaml @@ -0,0 +1,71 @@ +name: Merge Docker Images + +on: + workflow_call: + inputs: + cache_id: + required: true + type: string + extract_flavor: + required: false + default: "" + type: string + extract_tags: + required: false + default: "" + type: string + image_name: + required: true + type: string + registry: + required: false + default: ghcr.io + type: string + +env: + FULL_IMAGE_NAME: ${{ inputs.registry }}/${{ inputs.image_name }} + +jobs: + merge-images: + runs-on: ubuntu-latest + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-${{ inputs.cache_id }}-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + ${{ inputs.extract_tags }} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + ${{ inputs.extract_flavor }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} From 0a215304b233742c36a426bfe4646c86921b4bac Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 21 Oct 2024 10:20:47 -0400 Subject: [PATCH 3/4] fix: env vars in workflow --- .github/workflows/docker-build.yaml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index f376f148..5dab1b59 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -7,25 +7,20 @@ on: - main - dev -env: - CUDA_IMAGE_TAG: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda - CUDA_IMAGE_FLAVOR: suffix=-cuda,onlatest=true - IMAGE_NAME: ${{ github.repository }} - jobs: build-main-image: uses: ./.github/workflows/build-docker-image.yaml with: - image_name: ${{ env.IMAGE_NAME }} + image_name: ${{ github.repository }} cache_id: main build-cuda-image: uses: ./.github/workflows/build-docker-image.yaml with: - image_name: ${{ env.IMAGE_NAME }} + image_name: ${{ github.repository }} cache_id: cuda - image_tag: ${{ env.CUDA_IMAGE_TAG }} - extract_flavor: ${{ env.CUDA_IMAGE_FLAVOR }} + image_tag: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda + extract_flavor: suffix=-cuda,onlatest=true build_args: | USE_CUDA=true @@ -33,14 +28,14 @@ jobs: uses: ./.github/workflows/merge-docker-images.yaml needs: [build-main-image] with: - image_name: ${{ env.IMAGE_NAME }} + image_name: ${{ github.repository }} cache_id: main merge-cuda-images: uses: ./.github/workflows/merge-docker-images.yaml needs: [build-cuda-image] with: - image_name: ${{ env.IMAGE_NAME }} + image_name: ${{ github.repository }} cache_id: cuda - extract_flavor: ${{ env.CUDA_IMAGE_FLAVOR }} - extract_tags: ${{ env.CUDA_IMAGE_TAG }} + extract_flavor: suffix=-cuda,onlatest=true + extract_tags: type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda From 6a63c0388eae700ff57fe1138eff9641c4f935de Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 21 Oct 2024 10:34:43 -0400 Subject: [PATCH 4/4] fix: dockerfile casing --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac5864da..5d2677df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm as base +FROM python:3.11-slim-bookworm AS base # Use args ARG USE_CUDA