[Infra] Migrate rest of linux builder workflows off GCP runners. (#18511

) This commit is part of this larger issue that is tracking our migration off the GCP runners, storage buckets, etc: #18238. This builds on #18381, which migrated * `linux_x86_64_release_packages` * `linux_x64_clang_debug` * `linux_x64_clang_tsan` Here, we move over the rest of the critical linux builder workflows off of the GCP runners: * `linux_x64_clang` * `linux_x64_clang_asan` This also drops all CI usage of the GCP cache (`http://storage.googleapis.com/iree-sccache/ccache`). Some workflows now use sccache backed by Azure Blob Storage as a replacement. There are few issues with this (mozilla/sccache#2258) that prevent us providing read only access to the cache in PRs created from forks, so **PRs from forks currently don't use the cache and will have slower builds**. We're covering for this slowdown by using larger runners, but if we can roll out caching to all builds then we might use runners with fewer cores. Along with the changes to the cache, usage of Docker is rebased on images in the https://github.com/iree-org/base-docker-images/ repo and the `build_tools/docker/docker_run.sh` script is now only used by unmigrated workflows (`linux_arm64_clang` and `build_test_all_bazel`). --------- Signed-off-by: saienduri <[email protected]> Signed-off-by: Elias Joseph <[email protected]> Co-authored-by: Scott Todd <[email protected]> Co-authored-by: Elias Joseph <[email protected]>
iree-org · Sep 13, 2024 · cc891ba · cc891ba
1 parent 27b0829
commit cc891ba
Show file tree

Hide file tree

Showing 14 changed files with 168 additions and 79 deletions.
diff --git a/.github/workflows/ci_linux_x64_clang.yml b/.github/workflows/ci_linux_x64_clang.yml
@@ -28,37 +28,45 @@ jobs:
   linux_x64_clang:
     needs: setup
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'linux_x64_clang')
-    runs-on:
-      - self-hosted # must come first
-      - runner-group=${{ needs.setup.outputs.runner-group }}
-      - environment=${{ needs.setup.outputs.runner-env }}
-      - cpu
-      - os-family=Linux
+    runs-on: azure-linux-scale
+    container:
+      image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
+    defaults:
+      run:
+        shell: bash
     env:
-      BUILD_DIR: full-build-dir
+      BUILD_DIR: build
+      SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}"
+      SCCACHE_AZURE_BLOB_CONTAINER: ccache-container
+      SCCACHE_CACHE_ZSTD_LEVEL: 10
+      SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang"
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           submodules: true
-      - name: "Building IREE"
-        env:
-          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+      - name: Install Python requirements
+        run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
+      - name: CMake - configure
         run: |
-          ./build_tools/github_actions/docker_run.sh \
-            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
-            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
-            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-            ./build_tools/cmake/build_all.sh "${BUILD_DIR}"
-      - name: "Testing IREE"
+          source ./build_tools/cmake/setup_sccache.sh
+          cmake \
+            -G Ninja \
+            -B ${BUILD_DIR} \
+            -DPython3_EXECUTABLE="$(which python3)" \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DIREE_BUILD_PYTHON_BINDINGS=ON \
+            -DIREE_ENABLE_LLD=ON \
+            -DIREE_ENABLE_ASSERTIONS=ON \
+            -DIREE_BUILD_DOCS=ON \
+            -DIREE_TARGET_BACKEND_WEBGPU_SPIRV=ON
+      - name: CMake - build
         run: |
-          ./build_tools/github_actions/docker_run.sh \
-            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-            ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"
-      - name: "Testing iree-dialects"
-        run: |
-          ./build_tools/github_actions/docker_run.sh \
-            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-            ./build_tools/cmake/test_iree_dialects.sh "${BUILD_DIR}"
+          cmake --build ${BUILD_DIR} -- -k 0
+          cmake --build ${BUILD_DIR} --target install -- -k 0
+          cmake --build ${BUILD_DIR} --target iree-test-deps -- -k 0
+          sccache --show-stats
+      - name: Run CTest
+        run: ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"
+      - name: Test iree-dialects
+        run: ./build_tools/cmake/test_iree_dialects.sh "${BUILD_DIR}"
diff --git a/.github/workflows/ci_linux_x64_clang_asan.yml b/.github/workflows/ci_linux_x64_clang_asan.yml
@@ -28,27 +28,28 @@ jobs:
   linux_x64_clang_asan:
     needs: setup
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'linux_x64_clang_asan')
-    runs-on:
-      - self-hosted # must come first
-      - runner-group=${{ needs.setup.outputs.runner-group }}
-      - environment=${{ needs.setup.outputs.runner-env }}
-      - cpu
-      - os-family=Linux
+    runs-on: azure-linux-scale
+    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
+    defaults:
+      run:
+        shell: bash
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           submodules: true
-      - name: "Building and testing with ASan"
+      - name: Install Python requirements
+        run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
+      - name: Build and test with ASan
         env:
-          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+          # Use a modern clang explicitly.
+          CC: clang-19
+          CXX: clang++-19
+          SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}"
+          SCCACHE_AZURE_BLOB_CONTAINER: ccache-container
+          SCCACHE_CACHE_ZSTD_LEVEL: 10
+          SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_asan"
         run: |
-          # Note that this uses the latest version of the clang compiler, etc.
-          # This gives us access to the latest features and validates that IREE
-          # builds using the latest versions.
-          ./build_tools/github_actions/docker_run.sh \
-            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
-            gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
-            ./build_tools/cmake/build_and_test_asan.sh
+          source build_tools/cmake/setup_sccache.sh
+          ./build_tools/cmake/build_and_test_asan.sh
+          sccache --show-stats
diff --git a/.github/workflows/ci_linux_x64_clang_byollvm.yml b/.github/workflows/ci_linux_x64_clang_byollvm.yml
@@ -8,8 +8,6 @@ name: CI - Linux x64 clang BYO LLVM
 
 on:
   pull_request:
-    branches:
-      - main
     paths:
       - ".github/workflows/ci_linux_x64_clang_byollvm.yml"
   schedule:
@@ -28,7 +26,7 @@ concurrency:
 jobs:
   linux_x64_clang_byollvm:
     runs-on: ubuntu-20.04
-    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4
+    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
     defaults:
       run:
         shell: bash

diff --git a/.github/workflows/ci_linux_x64_clang_debug.yml b/.github/workflows/ci_linux_x64_clang_debug.yml
@@ -8,8 +8,6 @@ name: CI - Linux x64 clang debug
 
 on:
   pull_request:
-    branches:
-      - main
     paths:
       - ".github/workflows/ci_linux_x64_clang_debug.yml"
   schedule:
@@ -34,23 +32,26 @@ jobs:
   linux_x64_clang_debug:
     needs: setup
     runs-on: azure-linux-scale
-    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4
+    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
     defaults:
       run:
         shell: bash
     env:
       BUILD_DIR: build-debug
+      SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}"
+      SCCACHE_AZURE_BLOB_CONTAINER: ccache-container
+      SCCACHE_CACHE_ZSTD_LEVEL: 10
+      SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_debug"
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           submodules: true
       - name: Install Python requirements
         run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
-      # Note: Not using ccache here. Debug builds need a lot of cache space
-      # and caching only provides marginal build time improvements.
       - name: CMake - configure
         run: |
+          source ./build_tools/cmake/setup_sccache.sh
           cmake \
             -G Ninja \
             -B ${BUILD_DIR} \
@@ -60,7 +61,9 @@ jobs:
             -DIREE_ENABLE_LLD=ON \
             -DIREE_ENABLE_ASSERTIONS=ON
       - name: CMake - build
-        run: cmake --build ${BUILD_DIR} -- -k 0
+        run: |
+          cmake --build ${BUILD_DIR} -- -k 0
+          sccache --show-stats
       # We could build `iree-test-deps` or run some unit tests here, but the
       # main thing we want coverage for is the build itself and those steps
       # would add 10+ minutes to the job.
diff --git a/.github/workflows/ci_linux_x64_clang_tsan.yml b/.github/workflows/ci_linux_x64_clang_tsan.yml
@@ -8,8 +8,6 @@ name: CI - Linux x64 clang TSan
 
 on:
   pull_request:
-    branches:
-      - main
     paths:
       - ".github/workflows/ci_linux_x64_clang_tsan.yml"
   schedule:
@@ -32,21 +30,34 @@ jobs:
   linux_x64_clang_tsan:
     needs: setup
     runs-on: azure-linux-scale
+    container:
+      image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
+      # TSan in particular needs some settings that this option includes:
+      #   * https://github.com/google/sanitizers/issues/1716
+      #   * https://security.stackexchange.com/q/214923
+      # Note that we are not using Docker for sandboxing. If we were, this
+      # would *not* be safe.
+      options: --privileged
+    defaults:
+      run:
+        shell: bash
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           submodules: true
-      - name: "Building and testing with TSan"
+      - name: Install Python requirements
+        run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
+      - name: Build and test with TSan
         env:
-          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+          # Use a modern clang explicitly.
+          CC: clang-19
+          CXX: clang++-19
+          SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}"
+          SCCACHE_AZURE_BLOB_CONTAINER: ccache-container
+          SCCACHE_CACHE_ZSTD_LEVEL: 10
+          SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_tsan"
         run: |
-          # Note that this uses the latest version of the clang compiler, etc.
-          # This gives us access to the latest features and validates that IREE
-          # builds using the latest versions.
-          ./build_tools/github_actions/docker_run.sh \
-            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
-            gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
-            ./build_tools/cmake/build_and_test_tsan.sh
+          source build_tools/cmake/setup_sccache.sh
+          ./build_tools/cmake/build_and_test_tsan.sh
+          sccache --show-stats
diff --git a/.github/workflows/ci_linux_x64_gcc.yml b/.github/workflows/ci_linux_x64_gcc.yml
@@ -8,8 +8,6 @@ name: CI - Linux x64 gcc
 
 on:
   pull_request:
-    branches:
-      - main
     paths:
       - ".github/workflows/ci_linux_x64_gcc.yml"
   schedule:
@@ -28,7 +26,7 @@ concurrency:
 jobs:
   linux_x64_gcc:
     runs-on: ubuntu-20.04
-    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4
+    container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
     defaults:
       run:
         shell: bash

diff --git a/build_tools/cmake/build_and_test_asan.sh b/build_tools/cmake/build_and_test_asan.sh
@@ -28,7 +28,6 @@ IREE_TARGET_BACKEND_CUDA="${IREE_TARGET_BACKEND_CUDA:-${OFF_IF_DARWIN}}"
 IREE_TARGET_BACKEND_ROCM="${IREE_TARGET_BACKEND_ROCM:-${OFF_IF_DARWIN}}"
 
 source build_tools/cmake/setup_build.sh
-source build_tools/cmake/setup_ccache.sh
 
 CMAKE_ARGS=(
   "-G" "Ninja"
@@ -38,7 +37,8 @@ CMAKE_ARGS=(
   "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
   "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
   "-DIREE_ENABLE_ASSERTIONS=${IREE_ENABLE_ASSERTIONS}"
-
+  "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+  "-DCMAKE_C_COMPILER_LAUNCHER=sccache"
   "-DIREE_HAL_DRIVER_CUDA=${IREE_HAL_DRIVER_CUDA}"
   "-DIREE_HAL_DRIVER_HIP=${IREE_HAL_DRIVER_HIP}"
   "-DIREE_TARGET_BACKEND_CUDA=${IREE_TARGET_BACKEND_CUDA}"
@@ -57,10 +57,6 @@ echo "::group::Building test deps"
 "${CMAKE_BIN?}" --build "${BUILD_DIR?}" --target iree-test-deps -- -k 0
 echo "::endgroup::"
 
-if (( IREE_USE_CCACHE == 1 )); then
-  ccache --show-stats
-fi
-
 # Respect the user setting, but default to as many jobs as we have cores.
 export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
 

diff --git a/build_tools/cmake/build_and_test_tsan.sh b/build_tools/cmake/build_and_test_tsan.sh
@@ -26,7 +26,6 @@ IREE_TARGET_BACKEND_CUDA="${IREE_TARGET_BACKEND_CUDA:-${OFF_IF_DARWIN}}"
 IREE_TARGET_BACKEND_ROCM="${IREE_TARGET_BACKEND_ROCM:-${OFF_IF_DARWIN}}"
 
 source build_tools/cmake/setup_build.sh
-source build_tools/cmake/setup_ccache.sh
 
 CMAKE_ARGS=(
   "-G" "Ninja"
@@ -43,10 +42,19 @@ CMAKE_ARGS=(
   # Enable TSan in all C/C++ targets, including IREE runtime, compiler, tests.
   "-DIREE_ENABLE_TSAN=ON"
 
+
   "-DIREE_HAL_DRIVER_CUDA=${IREE_HAL_DRIVER_CUDA}"
   "-DIREE_HAL_DRIVER_HIP=${IREE_HAL_DRIVER_HIP}"
   "-DIREE_TARGET_BACKEND_CUDA=${IREE_TARGET_BACKEND_CUDA}"
   "-DIREE_TARGET_BACKEND_ROCM=${IREE_TARGET_BACKEND_ROCM}"
+
+  # Workaround for this weird issue:
+  # https://github.com/google/benchmark/issues/773#issuecomment-616067912
+  "-DRUN_HAVE_STD_REGEX=0"
+  "-DRUN_HAVE_POSIX_REGEX=0"
+  "-DCOMPILE_HAVE_GNU_POSIX_REGEX=0"
+  "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+  "-DCMAKE_C_COMPILER_LAUNCHER=sccache"
 )
 
 "${CMAKE_BIN}" -B "${BUILD_DIR}" "${CMAKE_ARGS[@]?}"
@@ -59,10 +67,6 @@ echo "Building test deps"
 echo "------------------"
 "$CMAKE_BIN" --build "${BUILD_DIR}" --target iree-test-deps -- -k 0
 
-if (( IREE_USE_CCACHE == 1 )); then
-  ccache --show-stats
-fi
-
 # Disable actually running GPU tests. This tends to yield TSan reports that are
 # specific to one's particular GPU driver and therefore hard to reproduce across
 # machines and often un-actionable anyway.

diff --git a/build_tools/cmake/setup_sccache.sh b/build_tools/cmake/setup_sccache.sh
@@ -0,0 +1,35 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# sccache (https://github.com/mozilla/sccache/) setup, focused on Azure Blob
+# Storage. See https://github.com/mozilla/sccache/blob/main/docs/Azure.md.
+#
+# If the `SCCACHE_AZURE_CONNECTION_STRING` environment variable is set, this
+# will enable sccache. Note that `SCCACHE_AZURE_BLOB_CONTAINER` should also be
+# set. The `SCCACHE_CACHE_ZSTD_LEVEL` and `SCCACHE_AZURE_KEY_PREFIX`
+# environment variables are also recommended. We could give them default values
+# here if we wanted.
+#
+# If the `SCCACHE_AZURE_CONNECTION_STRING` environment variable is _not_ set,
+# this keeps sccache disabled. It does _not_ use a readonly cache.
+#
+# Note: this file must be *sourced* not executed.
+
+set -eo pipefail
+
+if [ -n "$SCCACHE_AZURE_CONNECTION_STRING" ]; then
+  echo "Connection string set, using sccache"
+  export IREE_USE_SCCACHE=1
+  export CMAKE_C_COMPILER_LAUNCHER="$(which sccache)"
+  export CMAKE_CXX_COMPILER_LAUNCHER="$(which sccache)"
+else
+  echo "Connection string _not_ set, skipping sccache setup"
+  unset SCCACHE_AZURE_CONNECTION_STRING
+  export IREE_USE_SCCACHE=0
+fi
+
+sccache --zero-stats
+sccache --show-stats
diff --git a/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel b/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel
@@ -14,6 +14,7 @@ package(
 
 iree_lit_test_suite(
     name = "lit",
+    timeout = "moderate",
     srcs = enforce_glob(
         [
             "compile_regressions.mlir",

diff --git a/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt b/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt
@@ -22,6 +22,8 @@ iree_lit_test_suite(
   TOOLS
     FileCheck
     iree-opt
+  TIMEOUT
+    300
 )
 
 ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###