Back out "Update the existing Build Images workflow to build all FBPC…

…S images and run on pull requests." (#2206) Summary: Pull Request resolved: #2206 Builds fail from PR created in forks. Backing out for now until I can resolve that. Reviewed By: jrodal98 Differential Revision: D44184136 fbshipit-source-id: ceedab8d1c0aad2e2a9ed340f4f2cd3b017091c5
facebookresearch · Mar 17, 2023 · 38ee166 · 38ee166
1 parent c8db2fa
commit 38ee166
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 83 deletions.
diff --git a/.github/workflows/build_binary_image.yml b/.github/workflows/build_binary_image.yml
@@ -31,7 +31,7 @@ on:
 env:
   DISTRO: ubuntu
   REGISTRY: ghcr.io
-  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/
+  RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/
 
 jobs:
   build_image:
@@ -44,14 +44,17 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Get Docker Image Name
-        id: get_docker_image_name
-        run: |
-          echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }}
+          restore-keys: |
+            ${{ runner.os }}-${{ inputs.image_to_build }}
+
       - name: Pull FBPCF Image
         run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
 
@@ -68,12 +71,22 @@ jobs:
           context: .
           file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu
           tags: |
-            ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }}
+            ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }}
           build-args: |
             tag=latest
             os_release=20.04
             fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }}
           push: ${{ inputs.push_to_registry }}
           load: ${{ inputs.load_image_locally }}
-          cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }}
-          cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      # This ugly bit is necessary or else our cache will grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix: T135482742
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.github/workflows/build_fbpcs_images.yml b/.github/workflows/build_fbpcs_images.yml
@@ -1,18 +1,12 @@
-name: Build, Test, and Publish the FBPCS Docker Images
+name: Build and Publish Data Processing and EMP Games Docker Images
 
 on:
   push:
     branches: [ main ]
-  pull_request:
-    branches: [ main ]
 
 env:
-  FBPCF_VERSION: 2.1.132  # Please also update line 25 in .github/workflows/docker-publish.yml
-  PID_VERSION: 0.0.8  # Please also update line 26 in .github/workflows/docker-publish.yml
+  FBPCF_VERSION: 2.1.132  # Please also update line 29 in .github/workflows/docker-publish.yml
   REGISTRY: ghcr.io
-  LOCAL_IMAGE_NAME: fbpcs/onedocker/test
-  RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc
-  RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker
 
 jobs:
   output_version:
@@ -24,12 +18,6 @@ jobs:
       - id: set_version
         run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT
 
-  build_coordinator_image:
-    name: Build the Coordinator image
-    uses: ./.github/workflows/coordinator-publish.yml
-    with:
-      new_tag: ${{ github.sha }}
-
   build_and_publish_data_processing_image:
     name: Build and Publish Data Processing Image
     needs: output_version
@@ -49,37 +37,3 @@ jobs:
       tag: latest
       fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}}
       push_to_registry: true
-
-  build_test_onedocker_image:
-    name: Build the bundled test version of the onedocker image
-    needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build Image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: docker/onedocker/test/Dockerfile.ubuntu
-          tags: |
-            ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }}
-          build-args: |
-            tag=${{ github.sha }}
-            os_release=20.04
-            private_id_tag=${{ env.PID_VERSION }}
-            repository_url=${{ env.RC_REGISTRY_URL }}
-          push: true
-          pull: true
-          cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }}
-          cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max
diff --git a/.github/workflows/coordinator-publish.yml b/.github/workflows/coordinator-publish.yml
@@ -3,6 +3,9 @@ name: Publish Coordinator Image
 on:
   workflow_dispatch:
     inputs:
+      name:
+        description: "Manually running this workflow to build a coordinator image"
+        default: "Run"
       new_tag:
         description: "The new tag of the docker image"
         required: false
@@ -11,18 +14,7 @@ on:
       tracker_hash:
         description: "[Internal usage] Used for tracking workflow job status within Meta infra"
         required: false
-        type: string
-  workflow_call:
-    inputs:
-      new_tag:
-        description: "The new tag of the docker image"
-        required: false
-        type: string
-        default: latest-build
-      tracker_hash:
-        description: "[Internal usage] Used for tracking workflow job status within Meta infra"
-        required: false
-        type: string
+        type: str
 
 env:
   DISTRO: ubuntu
@@ -39,29 +31,32 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout@v3
-
+      - uses: actions/checkout@v2
       - name: Print Tracker Hash
-        run: echo ${{ inputs.tracker_hash }}
+        run: echo ${{ github.event.inputs.tracker_hash }}
 
       - name: Build image
         run: |
-          docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} .
+          docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} .
 
       # Tests will be added here
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set output
+        id: vars
+        run: echo ::set-output name=ref::${GITHUB_REF##*/}
+
       - name: Tag docker image
         run: |
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
-          ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }}
-          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }}
+          docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }}
 
       - name: Push image to registry
         run: |

diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -22,8 +22,8 @@ env:
   PL_CONTAINER_NAME: e2e_pl_container
   PA_CONTAINER_NAME: e2e_pa_container
   TIME_RANGE: 24 hours
-  FBPCF_VERSION: 2.1.132  # Please also update line 10 in .github/workflows/build_fbpcs_images.yml
-  PID_VERSION: 0.0.8  # Please also update line 11 in .github/workflows/build_fbpcs_images.yml
+  FBPCF_VERSION: 2.1.132  # Please also update line 8 in .github/workflows/build_fbpcs_images.yml
+  PID_VERSION: 0.0.8
 
 jobs:
   ### Build and publish rc/onedocker image

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,18 @@
+name: Test building image
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  building_image:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build image
+      run: |
+        ./build-docker.sh onedocker
diff --git a/docker/onedocker/test/Dockerfile.ubuntu b/docker/onedocker/test/Dockerfile.ubuntu
@@ -5,10 +5,8 @@
 ARG os_release="20.04"
 ARG tag="latest"
 ARG private_id_tag="latest"
-# Use the local repository as default
-ARG repository_url="fbpcs"
-FROM ${repository_url}/data-processing:${tag} as data_processing
-FROM ${repository_url}/emp-games:${tag} as emp_games
+FROM fbpcs/data-processing:${tag} as data_processing
+FROM fbpcs/emp-games:${tag} as emp_games
 FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id
 
 FROM ubuntu:${os_release}