[Bench][CI] allow custom Compute Runtime from PRs

lukaszstolarczuk · lukaszstolarczuk · commit 394d50ce8dcd · 2026-01-29T10:38:12.000+01:00
diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml
@@ -137,6 +137,12 @@ on:
         type: string
         required: False
         default: '0'
+      benchmark_custom_cr:
+        description: |
+          Custom Compute Runtime to use in benchmarks.
+        type: string
+        required: False
+        default: ''
 
       in_workflow_call_mode:
         description: |
@@ -369,3 +375,4 @@ jobs:
         build_ref: ${{ inputs.repo_ref }}
         runner: ${{ inputs.runner }}
         gdb_mode: ${{ inputs.benchmark_gdb_mode }}
+        custom_cr: ${{ inputs.benchmark_custom_cr }}
diff --git a/.github/workflows/sycl-ur-perf-benchmarking.yml b/.github/workflows/sycl-ur-perf-benchmarking.yml
@@ -95,6 +95,11 @@ on:
           - '0'
           - '1'
         default: '0'
+      custom_cr:
+        description: Custom Compute Runtime to use in benchmarks.
+        type: string
+        required: false
+        default: ''
 
 concurrency:
   # Cancel a currently running workflow for:
@@ -119,6 +124,7 @@ jobs:
       COMMIT_HASH: ${{ inputs.commit_hash }}
       PR_NO: ${{ inputs.pr_no }}
       SAVE_NAME: ${{ inputs.save_name }}
+      CUSTOM_CR: ${{ inputs.custom_cr }}
     outputs:
       benchmark_save_name: ${{ steps.sanitize.outputs.benchmark_save_name }}
       build_ref: ${{ steps.sanitize.outputs.build_ref }}
@@ -140,6 +146,10 @@ jobs:
           validate_if_nonempty "$SAVE_NAME" '^[A-Za-z][A-Za-z0-9_-]+$' "Bad save name. Use only alphanumerics, dash, underscore, and start with a letter."
           validate_if_nonempty "$SAVE_NAME" '^(?![Bb]+aseline$)' "Save name cannot be 'Baseline' nor 'baseline' - it's reserved for nightly."
 
+          # Just validate, don't output this var. We pass unquoted string via inputs.custom_cr, as we actually
+          # quote it within the benchmark's action. Double quoting can produce unexpected issues.
+          validate_if_nonempty "$CUSTOM_CR" '^[A-Za-z0-9._/-]+$' "Custom Compute Runtime can be either a proper git ref or 'PR_<no.>'."
+
           BENCHMARK_SAVE_NAME=""
           BUILD_REF="$GITHUB_REF"
           if [ -n "$SAVE_NAME" ]; then
@@ -206,6 +216,7 @@ jobs:
       benchmark_preset: ${{ inputs.preset }}
       benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
       benchmark_gdb_mode: ${{ inputs.gdb_mode }}
+      benchmark_custom_cr: ${{ inputs.custom_cr }}
       repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
       toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }}
diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml
@@ -43,6 +43,13 @@ inputs:
   runner:
     type: string
     required: True
+  # It can be either a tag/branch/commit or a PR_<number>.
+  # If the latter is given, latest IGC release will be installed from debs.
+  # This assumes that the PR changes were based on the latest compute_runtime version.
+  custom_cr:
+    type: string
+    required: False
+    default: ""
 
 runs:
   # composite actions don't make use of 'name', so copy-paste steps' names as a comment in the first line of each step
@@ -150,6 +157,8 @@ runs:
   - name: Establish results branch, repo path, and workdir
     id: establish_outputs
     shell: bash
+    env:
+      CUSTOM_CR: ${{ inputs.custom_cr }}
     run: |
       # Establish results branch, repo path, and workdir
       #
@@ -164,6 +173,17 @@ runs:
 
       BENCHMARK_RESULTS_REPO_PATH="$(realpath ./llvm-ci-perf-results)"
       echo "BENCHMARK_RESULTS_REPO_PATH=$BENCHMARK_RESULTS_REPO_PATH" >> $GITHUB_OUTPUT
+
+      CR_BUILD_REF=""
+      if [ -n "${CUSTOM_CR}" ] && [[ "${CUSTOM_CR}" == PR_* ]]; then
+        pr_no="$(echo "${CUSTOM_CR}" | sed 's/PR_//')"
+        CR_BUILD_REF="refs/pull/$pr_no/head"
+        echo "Using custom compute runtime ref: ${CR_BUILD_REF}"
+      elif [ -n "${CUSTOM_CR}" ]; then
+        CR_BUILD_REF="${CUSTOM_CR}"
+        echo "Using custom compute runtime ref: ${CR_BUILD_REF}"
+      fi
+      echo "CR_BUILD_REF=${CR_BUILD_REF}" >> $GITHUB_OUTPUT
   - name: Checkout results repo
     uses: actions/checkout@v5
     with:
@@ -221,18 +241,10 @@ runs:
     shell: bash
     env:
       RUNNER_TAG: ${{ inputs.runner }}
+      GITHUB_TOKEN: ${{ github.token }}
+      CR_BUILD_REF: ${{ steps.establish_outputs.outputs.CR_BUILD_REF }}
     run: |
       # Install dependencies
-
-      echo "::group::use_compute_runtime_tag_cache"
-
-      # Cache the compute_runtime version from dependencies.json, but perform a
-      # check with L0 version before using it: This value is not guaranteed to
-      # accurately reflect the current compute_runtime version used, as the
-      # docker images are built nightly.
-      export COMPUTE_RUNTIME_TAG_CACHE="$(cat ./devops/dependencies.json | jq -r .linux.compute_runtime.github_tag)"
-
-      echo "::endgroup::"
       echo "::group::install_perf"
 
       # Install perf in version matching the host kernel.
@@ -248,6 +260,38 @@ runs:
       sudo apt-get update
       sudo apt-get install -y linux-tools-$(uname -r)
 
+      echo "::endgroup::"
+      echo "::group::install_igc"
+
+      # If a custom compute_runtime tag is provided and it points to a PR,
+      # install latest IGC release from debs.
+      echo "CR_BUILD_REF='${CR_BUILD_REF}'"
+      if [[ "${CR_BUILD_REF}" != "" ]] && [[ "${CR_BUILD_REF}" == refs/pull/* ]]; then
+        # Find latest IGC release, download debs, and install them
+        curl --request GET \
+          --url https://api.github.com/repos/intel/intel-graphics-compiler/releases/latest \
+          --header "Authorization: Bearer ${GITHUB_TOKEN}" \
+          --header "X-GitHub-Api-Version: 2022-11-28" > latest_igc_release.json
+
+        IGC_CORE_DEB_URL=$(cat latest_igc_release.json | jq -r '.assets[] | select(.name | test("intel-igc-core-[0-9]+.*amd64.deb")) | .browser_download_url')
+        IGC_CORE_DEVEL_DEB_URL=$(cat latest_igc_release.json | jq -r '.assets[] | select(.name | test("intel-igc-core-devel_[0-9]+.*amd64.deb")) | .browser_download_url')
+        echo "IGC_CORE_DEB_URL=$IGC_CORE_DEB_URL"
+        echo "IGC_CORE_DEVEL_DEB_URL=$IGC_CORE_DEVEL_DEB_URL"
+
+        mkdir neo_debs || true
+        cd neo_debs
+        wget -q "$IGC_CORE_DEB_URL"
+        wget -q "$IGC_CORE_DEVEL_DEB_URL"
+        sudo dpkg -i --force-all *.deb
+        cd ..
+      elif [[ "${CR_BUILD_REF}" == "" ]]; then
+        # Cache the compute_runtime version from dependencies.json, but perform a
+        # check with L0 version before using it: This value is not guaranteed to
+        # accurately reflect the current compute_runtime version used, as the
+        # docker images are built nightly.
+        export COMPUTE_RUNTIME_TAG_CACHE="$(cat ./devops/dependencies.json | jq -r .linux.compute_runtime.github_tag)"
+      fi
+
       echo "::endgroup::"
       echo "::group::install_python_deps"
 
@@ -272,6 +316,7 @@ runs:
       BENCH_WORKDIR: ${{ steps.establish_outputs.outputs.BENCH_WORKDIR }}
       BENCHMARK_RESULTS_REPO_PATH: ${{ steps.establish_outputs.outputs.BENCHMARK_RESULTS_REPO_PATH }}
       LLVM_BENCHMARKS_USE_GDB: ${{ inputs.gdb_mode }}
+      CR_BUILD_REF: ${{ steps.establish_outputs.outputs.CR_BUILD_REF }}
     run: |
       # Build and run benchmarks
 
@@ -298,6 +343,7 @@ runs:
         --timestamp-override "$SAVE_TIMESTAMP" \
         --detect-version sycl,compute_runtime \
         --produce-github-summary \
+        $([[ "${CR_BUILD_REF}" != "" ]] && echo "--compute-runtime ${CR_BUILD_REF}" || echo '') \
         ${{ inputs.exit_on_failure == 'true' && '--exit-on-failure --iterations 1' || '' }}
       # TODO: add back: "--flamegraph inclusive" once works properly
 
diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2025 Intel Corporation
+# Copyright (C) 2025-2026 Intel Corporation
 # Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
@@ -120,54 +120,55 @@ def _can_shallow_clone_ref(self, ref: str) -> bool:
             if output:
                 # Found the ref as a branch or tag
                 log.debug(
-                    f"Ref {ref} found as branch/tag via ls-remote, can shallow clone"
+                    f"Ref '{ref}' found as branch/tag via ls-remote, can shallow clone"
                 )
                 return True
             else:
-                # Not found as branch/tag, likely a SHA commit
+                # Not found as branch/tag, likely a SHA commit or a special ref
                 log.debug(
-                    f"Ref {ref} not found as branch/tag via ls-remote, likely SHA commit"
+                    f"Ref '{ref}' not found as branch/tag via ls-remote, likely SHA commit or a special ref"
                 )
                 return False
         except Exception as e:
             log.debug(
-                f"Could not check ref {ref} via ls-remote: {e}, assuming SHA commit"
+                f"Could not check ref '{ref}' via ls-remote: {e}, assuming SHA commit"
             )
             return False
 
     def _git_clone(self) -> None:
         """Clone the git repository."""
         try:
-            log.debug(f"Cloning {self._url} into {self.src_dir} at commit {self._ref}")
-            git_clone_cmd = f"git clone --recursive {self._url} {self.src_dir}"
+            log.debug(f"Cloning {self._url} into {self.src_dir} at ref {self._ref}")
+            git_clone_cmd = f"git clone --recursive --depth 1 {self._url} {self.src_dir}"
             if self._shallow_clone:
                 if self._can_shallow_clone_ref(self._ref):
                     # Shallow clone for branches and tags only
                     git_clone_cmd = f"git clone --recursive --depth 1 --branch {self._ref} {self._url} {self.src_dir}"
                 else:
-                    log.debug(f"Cannot shallow clone SHA {self._ref}, using full clone")
+                    log.debug(f"Cannot shallow clone ref '{self._ref}', clone default branch")
 
             run(git_clone_cmd)
-            run(f"git checkout {self._ref}", cwd=self.src_dir)
-            log.debug(f"Cloned {self._url} into {self.src_dir} at commit {self._ref}")
+            run(f"git fetch {self._url} {self._ref}", cwd=self.src_dir)
+            run(f"git checkout FETCH_HEAD", cwd=self.src_dir)
+            log.debug(f"Cloned {self._url} into {self.src_dir} at ref {self._ref}")
         except Exception as e:
             log.error(f"Failed to clone repository {self._url}: {e}")
             raise
 
     def _git_fetch(self) -> None:
-        """Fetch the latest changes from the remote repository."""
+        """Fetch the ref from the remote repository."""
         try:
-            log.debug(f"Fetching latest changes for {self._url} in {self.src_dir}")
-            run("git fetch", cwd=self.src_dir)
+            log.debug(f"Fetching ref '{self._ref}' for {self._url} in {self.src_dir}")
             run("git reset --hard", cwd=self.src_dir)
-            run(f"git checkout {self._ref}", cwd=self.src_dir)
-            log.debug(f"Fetched latest changes for {self._url} in {self.src_dir}")
+            run(f"git fetch {self._url} {self._ref}", cwd=self.src_dir)
+            run(f"git checkout FETCH_HEAD", cwd=self.src_dir)
+            log.debug(f"Fetched changes for {self._url} in {self.src_dir}")
         except Exception as e:
             log.error(f"Failed to fetch updates for repository {self._url}: {e}")
             raise
 
     def _setup_repo(self) -> bool:
-        """Clone a git repository into a specified directory at a specific commit.
+        """Clone a git repository into a specified directory at a specific ref.
         Returns:
             bool: True if the repository was cloned or updated, False if it was already up-to-date.
         """
