diff --git a/.github/workflows/ci_linux_x64_clang.yml b/.github/workflows/ci_linux_x64_clang.yml index 27fcbac87618..6cc0715b5bc7 100644 --- a/.github/workflows/ci_linux_x64_clang.yml +++ b/.github/workflows/ci_linux_x64_clang.yml @@ -28,37 +28,45 @@ jobs: linux_x64_clang: needs: setup if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'linux_x64_clang') - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=${{ needs.setup.outputs.runner-env }} - - cpu - - os-family=Linux + runs-on: azure-linux-scale + container: + image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 + defaults: + run: + shell: bash env: - BUILD_DIR: full-build-dir + BUILD_DIR: build + SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}" + SCCACHE_AZURE_BLOB_CONTAINER: ccache-container + SCCACHE_CACHE_ZSTD_LEVEL: 10 + SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang" steps: - name: "Checking out repository" uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: true - - name: "Building IREE" - env: - IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} + - name: Install Python requirements + run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt + - name: CMake - configure run: | - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \ - --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \ - gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - ./build_tools/cmake/build_all.sh "${BUILD_DIR}" - - name: "Testing IREE" + source ./build_tools/cmake/setup_sccache.sh + cmake \ + -G Ninja \ + -B ${BUILD_DIR} \ + -DPython3_EXECUTABLE="$(which python3)" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DIREE_BUILD_PYTHON_BINDINGS=ON \ + -DIREE_ENABLE_LLD=ON \ + -DIREE_ENABLE_ASSERTIONS=ON \ + -DIREE_BUILD_DOCS=ON \ + -DIREE_TARGET_BACKEND_WEBGPU_SPIRV=ON + - name: CMake - build run: | - ./build_tools/github_actions/docker_run.sh \ - gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}" - - name: "Testing iree-dialects" - run: | - ./build_tools/github_actions/docker_run.sh \ - gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - ./build_tools/cmake/test_iree_dialects.sh "${BUILD_DIR}" + cmake --build ${BUILD_DIR} -- -k 0 + cmake --build ${BUILD_DIR} --target install -- -k 0 + cmake --build ${BUILD_DIR} --target iree-test-deps -- -k 0 + sccache --show-stats + - name: Run CTest + run: ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}" + - name: Test iree-dialects + run: ./build_tools/cmake/test_iree_dialects.sh "${BUILD_DIR}" diff --git a/.github/workflows/ci_linux_x64_clang_asan.yml b/.github/workflows/ci_linux_x64_clang_asan.yml index b16818db6f5d..9731f8103007 100644 --- a/.github/workflows/ci_linux_x64_clang_asan.yml +++ b/.github/workflows/ci_linux_x64_clang_asan.yml @@ -28,27 +28,28 @@ jobs: linux_x64_clang_asan: needs: setup if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'linux_x64_clang_asan') - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=${{ needs.setup.outputs.runner-env }} - - cpu - - os-family=Linux + runs-on: azure-linux-scale + container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 + defaults: + run: + shell: bash steps: - name: "Checking out repository" uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: true - - name: "Building and testing with ASan" + - name: Install Python requirements + run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt + - name: Build and test with ASan env: - IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} + # Use a modern clang explicitly. + CC: clang-19 + CXX: clang++-19 + SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}" + SCCACHE_AZURE_BLOB_CONTAINER: ccache-container + SCCACHE_CACHE_ZSTD_LEVEL: 10 + SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_asan" run: | - # Note that this uses the latest version of the clang compiler, etc. - # This gives us access to the latest features and validates that IREE - # builds using the latest versions. - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \ - gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \ - ./build_tools/cmake/build_and_test_asan.sh + source build_tools/cmake/setup_sccache.sh + ./build_tools/cmake/build_and_test_asan.sh + sccache --show-stats diff --git a/.github/workflows/ci_linux_x64_clang_byollvm.yml b/.github/workflows/ci_linux_x64_clang_byollvm.yml index 11c413144543..3e20a8afe017 100644 --- a/.github/workflows/ci_linux_x64_clang_byollvm.yml +++ b/.github/workflows/ci_linux_x64_clang_byollvm.yml @@ -8,8 +8,6 @@ name: CI - Linux x64 clang BYO LLVM on: pull_request: - branches: - - main paths: - ".github/workflows/ci_linux_x64_clang_byollvm.yml" schedule: @@ -28,7 +26,7 @@ concurrency: jobs: linux_x64_clang_byollvm: runs-on: ubuntu-20.04 - container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4 + container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 defaults: run: shell: bash diff --git a/.github/workflows/ci_linux_x64_clang_debug.yml b/.github/workflows/ci_linux_x64_clang_debug.yml index ddc25aaae70e..0d8d409c2411 100644 --- a/.github/workflows/ci_linux_x64_clang_debug.yml +++ b/.github/workflows/ci_linux_x64_clang_debug.yml @@ -8,8 +8,6 @@ name: CI - Linux x64 clang debug on: pull_request: - branches: - - main paths: - ".github/workflows/ci_linux_x64_clang_debug.yml" schedule: @@ -34,12 +32,16 @@ jobs: linux_x64_clang_debug: needs: setup runs-on: azure-linux-scale - container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4 + container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 defaults: run: shell: bash env: BUILD_DIR: build-debug + SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}" + SCCACHE_AZURE_BLOB_CONTAINER: ccache-container + SCCACHE_CACHE_ZSTD_LEVEL: 10 + SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_debug" steps: - name: "Checking out repository" uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -47,10 +49,9 @@ jobs: submodules: true - name: Install Python requirements run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt - # Note: Not using ccache here. Debug builds need a lot of cache space - # and caching only provides marginal build time improvements. - name: CMake - configure run: | + source ./build_tools/cmake/setup_sccache.sh cmake \ -G Ninja \ -B ${BUILD_DIR} \ @@ -60,7 +61,9 @@ jobs: -DIREE_ENABLE_LLD=ON \ -DIREE_ENABLE_ASSERTIONS=ON - name: CMake - build - run: cmake --build ${BUILD_DIR} -- -k 0 + run: | + cmake --build ${BUILD_DIR} -- -k 0 + sccache --show-stats # We could build `iree-test-deps` or run some unit tests here, but the # main thing we want coverage for is the build itself and those steps # would add 10+ minutes to the job. diff --git a/.github/workflows/ci_linux_x64_clang_tsan.yml b/.github/workflows/ci_linux_x64_clang_tsan.yml index 63be367f64be..9ebf3cdb9f40 100644 --- a/.github/workflows/ci_linux_x64_clang_tsan.yml +++ b/.github/workflows/ci_linux_x64_clang_tsan.yml @@ -8,8 +8,6 @@ name: CI - Linux x64 clang TSan on: pull_request: - branches: - - main paths: - ".github/workflows/ci_linux_x64_clang_tsan.yml" schedule: @@ -32,21 +30,34 @@ jobs: linux_x64_clang_tsan: needs: setup runs-on: azure-linux-scale + container: + image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 + # TSan in particular needs some settings that this option includes: + # * https://github.com/google/sanitizers/issues/1716 + # * https://security.stackexchange.com/q/214923 + # Note that we are not using Docker for sandboxing. If we were, this + # would *not* be safe. + options: --privileged + defaults: + run: + shell: bash steps: - name: "Checking out repository" uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: true - - name: "Building and testing with TSan" + - name: Install Python requirements + run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt + - name: Build and test with TSan env: - IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} + # Use a modern clang explicitly. + CC: clang-19 + CXX: clang++-19 + SCCACHE_AZURE_CONNECTION_STRING: "${{ secrets.AZURE_CCACHE_CONNECTION_STRING }}" + SCCACHE_AZURE_BLOB_CONTAINER: ccache-container + SCCACHE_CACHE_ZSTD_LEVEL: 10 + SCCACHE_AZURE_KEY_PREFIX: "ci_linux_x64_clang_tsan" run: | - # Note that this uses the latest version of the clang compiler, etc. - # This gives us access to the latest features and validates that IREE - # builds using the latest versions. - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \ - gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \ - ./build_tools/cmake/build_and_test_tsan.sh + source build_tools/cmake/setup_sccache.sh + ./build_tools/cmake/build_and_test_tsan.sh + sccache --show-stats diff --git a/.github/workflows/ci_linux_x64_gcc.yml b/.github/workflows/ci_linux_x64_gcc.yml index 0256913619ff..db4ebcbcfc8e 100644 --- a/.github/workflows/ci_linux_x64_gcc.yml +++ b/.github/workflows/ci_linux_x64_gcc.yml @@ -8,8 +8,6 @@ name: CI - Linux x64 gcc on: pull_request: - branches: - - main paths: - ".github/workflows/ci_linux_x64_gcc.yml" schedule: @@ -28,7 +26,7 @@ concurrency: jobs: linux_x64_gcc: runs-on: ubuntu-20.04 - container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:54d9d17a79caa083aeff1243b27e767df2629b533c26e0b65d41beb160f197e4 + container: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 defaults: run: shell: bash diff --git a/build_tools/cmake/build_and_test_asan.sh b/build_tools/cmake/build_and_test_asan.sh index 5af05acc830d..05109c24a6b3 100755 --- a/build_tools/cmake/build_and_test_asan.sh +++ b/build_tools/cmake/build_and_test_asan.sh @@ -28,7 +28,6 @@ IREE_TARGET_BACKEND_CUDA="${IREE_TARGET_BACKEND_CUDA:-${OFF_IF_DARWIN}}" IREE_TARGET_BACKEND_ROCM="${IREE_TARGET_BACKEND_ROCM:-${OFF_IF_DARWIN}}" source build_tools/cmake/setup_build.sh -source build_tools/cmake/setup_ccache.sh CMAKE_ARGS=( "-G" "Ninja" @@ -38,7 +37,8 @@ CMAKE_ARGS=( "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" "-DIREE_ENABLE_ASSERTIONS=${IREE_ENABLE_ASSERTIONS}" - + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + "-DCMAKE_C_COMPILER_LAUNCHER=sccache" "-DIREE_HAL_DRIVER_CUDA=${IREE_HAL_DRIVER_CUDA}" "-DIREE_HAL_DRIVER_HIP=${IREE_HAL_DRIVER_HIP}" "-DIREE_TARGET_BACKEND_CUDA=${IREE_TARGET_BACKEND_CUDA}" @@ -57,10 +57,6 @@ echo "::group::Building test deps" "${CMAKE_BIN?}" --build "${BUILD_DIR?}" --target iree-test-deps -- -k 0 echo "::endgroup::" -if (( IREE_USE_CCACHE == 1 )); then - ccache --show-stats -fi - # Respect the user setting, but default to as many jobs as we have cores. export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)} diff --git a/build_tools/cmake/build_and_test_tsan.sh b/build_tools/cmake/build_and_test_tsan.sh index 57b0a707402d..c272a8ff5b01 100755 --- a/build_tools/cmake/build_and_test_tsan.sh +++ b/build_tools/cmake/build_and_test_tsan.sh @@ -26,7 +26,6 @@ IREE_TARGET_BACKEND_CUDA="${IREE_TARGET_BACKEND_CUDA:-${OFF_IF_DARWIN}}" IREE_TARGET_BACKEND_ROCM="${IREE_TARGET_BACKEND_ROCM:-${OFF_IF_DARWIN}}" source build_tools/cmake/setup_build.sh -source build_tools/cmake/setup_ccache.sh CMAKE_ARGS=( "-G" "Ninja" @@ -43,10 +42,19 @@ CMAKE_ARGS=( # Enable TSan in all C/C++ targets, including IREE runtime, compiler, tests. "-DIREE_ENABLE_TSAN=ON" + "-DIREE_HAL_DRIVER_CUDA=${IREE_HAL_DRIVER_CUDA}" "-DIREE_HAL_DRIVER_HIP=${IREE_HAL_DRIVER_HIP}" "-DIREE_TARGET_BACKEND_CUDA=${IREE_TARGET_BACKEND_CUDA}" "-DIREE_TARGET_BACKEND_ROCM=${IREE_TARGET_BACKEND_ROCM}" + + # Workaround for this weird issue: + # https://github.com/google/benchmark/issues/773#issuecomment-616067912 + "-DRUN_HAVE_STD_REGEX=0" + "-DRUN_HAVE_POSIX_REGEX=0" + "-DCOMPILE_HAVE_GNU_POSIX_REGEX=0" + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache" + "-DCMAKE_C_COMPILER_LAUNCHER=sccache" ) "${CMAKE_BIN}" -B "${BUILD_DIR}" "${CMAKE_ARGS[@]?}" @@ -59,10 +67,6 @@ echo "Building test deps" echo "------------------" "$CMAKE_BIN" --build "${BUILD_DIR}" --target iree-test-deps -- -k 0 -if (( IREE_USE_CCACHE == 1 )); then - ccache --show-stats -fi - # Disable actually running GPU tests. This tends to yield TSan reports that are # specific to one's particular GPU driver and therefore hard to reproduce across # machines and often un-actionable anyway. diff --git a/build_tools/cmake/setup_sccache.sh b/build_tools/cmake/setup_sccache.sh new file mode 100644 index 000000000000..a71845051981 --- /dev/null +++ b/build_tools/cmake/setup_sccache.sh @@ -0,0 +1,35 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# sccache (https://github.com/mozilla/sccache/) setup, focused on Azure Blob +# Storage. See https://github.com/mozilla/sccache/blob/main/docs/Azure.md. +# +# If the `SCCACHE_AZURE_CONNECTION_STRING` environment variable is set, this +# will enable sccache. Note that `SCCACHE_AZURE_BLOB_CONTAINER` should also be +# set. The `SCCACHE_CACHE_ZSTD_LEVEL` and `SCCACHE_AZURE_KEY_PREFIX` +# environment variables are also recommended. We could give them default values +# here if we wanted. +# +# If the `SCCACHE_AZURE_CONNECTION_STRING` environment variable is _not_ set, +# this keeps sccache disabled. It does _not_ use a readonly cache. +# +# Note: this file must be *sourced* not executed. + +set -eo pipefail + +if [ -n "$SCCACHE_AZURE_CONNECTION_STRING" ]; then + echo "Connection string set, using sccache" + export IREE_USE_SCCACHE=1 + export CMAKE_C_COMPILER_LAUNCHER="$(which sccache)" + export CMAKE_CXX_COMPILER_LAUNCHER="$(which sccache)" +else + echo "Connection string _not_ set, skipping sccache setup" + unset SCCACHE_AZURE_CONNECTION_STRING + export IREE_USE_SCCACHE=0 +fi + +sccache --zero-stats +sccache --show-stats diff --git a/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel b/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel index 999a32f30922..428ca152d8ac 100644 --- a/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel +++ b/compiler/src/iree/compiler/ConstEval/test/BUILD.bazel @@ -14,6 +14,7 @@ package( iree_lit_test_suite( name = "lit", + timeout = "moderate", srcs = enforce_glob( [ "compile_regressions.mlir", diff --git a/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt b/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt index 247ae439d7cb..c7250cbc2915 100644 --- a/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt +++ b/compiler/src/iree/compiler/ConstEval/test/CMakeLists.txt @@ -22,6 +22,8 @@ iree_lit_test_suite( TOOLS FileCheck iree-opt + TIMEOUT + 300 ) ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ### diff --git a/docs/website/docs/developers/debugging/sanitizers.md b/docs/website/docs/developers/debugging/sanitizers.md index e1261d73400e..bfe5ddc6b784 100644 --- a/docs/website/docs/developers/debugging/sanitizers.md +++ b/docs/website/docs/developers/debugging/sanitizers.md @@ -155,6 +155,35 @@ ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-12/bin/llvm-symbolizer \ ### TSan (ThreadSanitizer) +To enable TSan: + +```shell +cmake -DIREE_ENABLE_TSAN=ON ... +``` + +You may also need: + +* Depending on your system (see + ): + + ```shell + -DRUN_HAVE_STD_REGEX=0 \ + -DRUN_HAVE_POSIX_REGEX=0 \ + -DCOMPILE_HAVE_GNU_POSIX_REGEX=0 \ + ``` + +* For clang < 18.1.0 on system with `vm.mmap_rnd_bits` > 28 (see + ): + + ```shell + sudo sysctl vm.mmap_rnd_bits=28 + ``` + + TSan in LLVM >= 18.1.0 supports 30 bits of ASLR entropy. If the layout is + unsupported, TSan will automatically re-execute without ASLR. + +* If running under Docker, add `--privileged` to your `docker run` command + #### C++ Standard Library with TSan support For best results to avoid false positives/negatives TSan needs all userspace diff --git a/tools/test/BUILD.bazel b/tools/test/BUILD.bazel index 46709aab8162..936990276a3b 100644 --- a/tools/test/BUILD.bazel +++ b/tools/test/BUILD.bazel @@ -16,6 +16,7 @@ package( iree_lit_test_suite( name = "lit", + timeout = "moderate", srcs = enforce_glob( [ "benchmark_flags.txt", diff --git a/tools/test/CMakeLists.txt b/tools/test/CMakeLists.txt index 952b8947a960..f577ec643775 100644 --- a/tools/test/CMakeLists.txt +++ b/tools/test/CMakeLists.txt @@ -55,6 +55,8 @@ iree_lit_test_suite( LABELS "driver=local-task" "hostonly" + TIMEOUT + 300 ) ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###