From 38ee166b2cd961db16a4aff1d53bffc397e7d386 Mon Sep 17 00:00:00 2001 From: Brian Muse Date: Fri, 17 Mar 2023 15:21:28 -0700 Subject: [PATCH] Back out "Update the existing Build Images workflow to build all FBPCS images and run on pull requests." (#2206) Summary: Pull Request resolved: https://github.com/facebookresearch/fbpcs/pull/2206 Builds fail from PR created in forks. Backing out for now until I can resolve that. Reviewed By: jrodal98 Differential Revision: D44184136 fbshipit-source-id: ceedab8d1c0aad2e2a9ed340f4f2cd3b017091c5 --- .github/workflows/build_binary_image.yml | 31 ++++++++++---- .github/workflows/build_fbpcs_images.yml | 50 +---------------------- .github/workflows/coordinator-publish.yml | 35 +++++++--------- .github/workflows/docker-publish.yml | 4 +- .github/workflows/test.yml | 18 ++++++++ docker/onedocker/test/Dockerfile.ubuntu | 6 +-- 6 files changed, 61 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build_binary_image.yml b/.github/workflows/build_binary_image.yml index bfe696921..3a660aa0c 100644 --- a/.github/workflows/build_binary_image.yml +++ b/.github/workflows/build_binary_image.yml @@ -31,7 +31,7 @@ on: env: DISTRO: ubuntu REGISTRY: ghcr.io - RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/ + RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/ jobs: build_image: @@ -44,14 +44,17 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Get Docker Image Name - id: get_docker_image_name - run: | - echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }} + restore-keys: | + ${{ runner.os }}-${{ inputs.image_to_build }} + - name: Pull FBPCF Image run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} @@ -68,12 +71,22 @@ jobs: context: . file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu tags: | - ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }} + ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }} build-args: | tag=latest os_release=20.04 fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} push: ${{ inputs.push_to_registry }} load: ${{ inputs.load_image_locally }} - cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }} - cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + # This ugly bit is necessary or else our cache will grow forever + # until it hits GitHub's limit of 5GB. + # Temp fix: T135482742 + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/build_fbpcs_images.yml b/.github/workflows/build_fbpcs_images.yml index c9d686b9f..ebff2d42f 100644 --- a/.github/workflows/build_fbpcs_images.yml +++ b/.github/workflows/build_fbpcs_images.yml @@ -1,18 +1,12 @@ -name: Build, Test, and Publish the FBPCS Docker Images +name: Build and Publish Data Processing and EMP Games Docker Images on: push: branches: [ main ] - pull_request: - branches: [ main ] env: - FBPCF_VERSION: 2.1.132 # Please also update line 25 in .github/workflows/docker-publish.yml - PID_VERSION: 0.0.8 # Please also update line 26 in .github/workflows/docker-publish.yml + FBPCF_VERSION: 2.1.132 # Please also update line 29 in .github/workflows/docker-publish.yml REGISTRY: ghcr.io - LOCAL_IMAGE_NAME: fbpcs/onedocker/test - RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc - RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker jobs: output_version: @@ -24,12 +18,6 @@ jobs: - id: set_version run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT - build_coordinator_image: - name: Build the Coordinator image - uses: ./.github/workflows/coordinator-publish.yml - with: - new_tag: ${{ github.sha }} - build_and_publish_data_processing_image: name: Build and Publish Data Processing Image needs: output_version @@ -49,37 +37,3 @@ jobs: tag: latest fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}} push_to_registry: true - - build_test_onedocker_image: - name: Build the bundled test version of the onedocker image - needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build Image - uses: docker/build-push-action@v3 - with: - context: . - file: docker/onedocker/test/Dockerfile.ubuntu - tags: | - ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }} - build-args: | - tag=${{ github.sha }} - os_release=20.04 - private_id_tag=${{ env.PID_VERSION }} - repository_url=${{ env.RC_REGISTRY_URL }} - push: true - pull: true - cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }} - cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max diff --git a/.github/workflows/coordinator-publish.yml b/.github/workflows/coordinator-publish.yml index ac11e920d..29a820f15 100644 --- a/.github/workflows/coordinator-publish.yml +++ b/.github/workflows/coordinator-publish.yml @@ -3,6 +3,9 @@ name: Publish Coordinator Image on: workflow_dispatch: inputs: + name: + description: "Manually running this workflow to build a coordinator image" + default: "Run" new_tag: description: "The new tag of the docker image" required: false @@ -11,18 +14,7 @@ on: tracker_hash: description: "[Internal usage] Used for tracking workflow job status within Meta infra" required: false - type: string - workflow_call: - inputs: - new_tag: - description: "The new tag of the docker image" - required: false - type: string - default: latest-build - tracker_hash: - description: "[Internal usage] Used for tracking workflow job status within Meta infra" - required: false - type: string + type: str env: DISTRO: ubuntu @@ -39,29 +31,32 @@ jobs: packages: write steps: - - uses: actions/checkout@v3 - + - uses: actions/checkout@v2 - name: Print Tracker Hash - run: echo ${{ inputs.tracker_hash }} + run: echo ${{ github.event.inputs.tracker_hash }} - name: Build image run: | - docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} . + docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} . # Tests will be added here - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set output + id: vars + run: echo ::set-output name=ref::${GITHUB_REF##*/} + - name: Tag docker image run: | - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} - ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }} - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} - name: Push image to registry run: | diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 79cf7d908..4bd9cc93d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,8 +22,8 @@ env: PL_CONTAINER_NAME: e2e_pl_container PA_CONTAINER_NAME: e2e_pa_container TIME_RANGE: 24 hours - FBPCF_VERSION: 2.1.132 # Please also update line 10 in .github/workflows/build_fbpcs_images.yml - PID_VERSION: 0.0.8 # Please also update line 11 in .github/workflows/build_fbpcs_images.yml + FBPCF_VERSION: 2.1.132 # Please also update line 8 in .github/workflows/build_fbpcs_images.yml + PID_VERSION: 0.0.8 jobs: ### Build and publish rc/onedocker image diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..b88ed2193 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +name: Test building image + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + building_image: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: | + ./build-docker.sh onedocker diff --git a/docker/onedocker/test/Dockerfile.ubuntu b/docker/onedocker/test/Dockerfile.ubuntu index 25fc4816c..683adaa95 100644 --- a/docker/onedocker/test/Dockerfile.ubuntu +++ b/docker/onedocker/test/Dockerfile.ubuntu @@ -5,10 +5,8 @@ ARG os_release="20.04" ARG tag="latest" ARG private_id_tag="latest" -# Use the local repository as default -ARG repository_url="fbpcs" -FROM ${repository_url}/data-processing:${tag} as data_processing -FROM ${repository_url}/emp-games:${tag} as emp_games +FROM fbpcs/data-processing:${tag} as data_processing +FROM fbpcs/emp-games:${tag} as emp_games FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id FROM ubuntu:${os_release}