[fusili] Dependency cleanup (remove IREE/LLVM/Catch2 source builds, bring in `lit` & `filecheck` standalone) #7415

Workflow file for this run

.github/workflows/ci-sharktank.yml at 012cc56

	# Copyright 2024 Advanced Micro Devices, Inc.
	#
	# Licensed under the Apache License v2.0 with LLVM Exceptions.
	# See https://llvm.org/LICENSE.txt for license information.
	# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

	name: CI - sharktank

	on:
	workflow_dispatch:
	pull_request:
	push:
	branches:
	- main

	permissions:
	contents: write
	pull-requests: write

	concurrency:
	# A PR number if a pull request and otherwise the commit hash. This cancels
	# queued and in-progress runs for the same PR (presubmit) or commit
	# (postsubmit). The workflow name is prepended to avoid conflicts between
	# different workflows.
	group: ${{ github.workflow }}-${{ github.event.number \|\| github.sha }}
	cancel-in-progress: true

	jobs:
	test:
	name: "Unit Tests (${{ matrix.runs-on }}, ${{ matrix.python-version }}, ${{ matrix.torch-version }})"
	strategy:
	matrix:
	python-version: ["3.11", "3.12"]
	torch-version: ["2.4.1", "2.5.1"]
	runs-on: [ubuntu-24.04]
	include:
	- runs-on: windows-2022
	python-version: "3.12"
	torch-version: "2.4.1"
	fail-fast: false
	runs-on: ${{matrix.runs-on}}
	defaults:
	run:
	shell: bash
	env:
	PIP_CACHE_DIR: "${{ github.workspace }}/.pip-cache"
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

	- name: "Setting up Python"
	id: setup_python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: ${{matrix.python-version}}

	- name: Cache Pip Packages
	uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
	id: cache-pip
	with:
	path: ${{ env.PIP_CACHE_DIR }}
	key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt','sharktank/requirements*.txt') }}

	- name: Install pip deps
	run: \|
	python -m pip install --no-compile --upgrade pip

	# Note: We install in three steps in order to satisfy requirements
	# from non default locations first.
	pip install --no-compile \
	--index-url https://download.pytorch.org/whl/cpu torch==${{matrix.torch-version}}+cpu
	pip install -r requirements-iree-pinned.txt
	pip install --no-compile \
	-r sharktank/requirements-tests.txt \
	-e sharktank/

	pip freeze

	- name: Run sharktank tests with coverage
	if: ${{ !cancelled() }}
	run: \|
	pytest -n 20 sharktank/ \
	--cov=sharktank \
	--cov-report xml:cov.xml \
	--cov-config=.coveragerc \
	--durations=10 \
	--log-cli-level=info \
	-v
	env:
	COVERAGE_FILE: .coverage.${{ matrix.python-version }}.${{ matrix.torch-version }}.${{ matrix.runs-on }}

	- name: Upload coverage data
	if: runner.os != 'Windows'
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
	with:
	name: coverage-data-${{ matrix.python-version }}-${{ matrix.torch-version }}-${{ matrix.runs-on }}
	path: .coverage.${{ matrix.python-version }}.${{ matrix.torch-version }}.${{ matrix.runs-on }}
	include-hidden-files: true

	- name: Upload coverage to Codecov
	if: ${{ github.event_name == 'pull_request' \|\| github.event_name == 'push' \|\| github.event_name == 'workflow_dispatch' \|\| github.event_name == 'schedule' }}
	uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24
	with:
	files: cov.xml

	coverage:
	name: Coverage
	runs-on: ubuntu-24.04
	needs: test
	permissions:
	pull-requests: write
	contents: write
	actions: read
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

	- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
	id: download
	with:
	pattern: coverage-*
	merge-multiple: true

	- name: Coverage comment
	id: coverage_comment
	uses: py-cov-action/python-coverage-comment-action@91910686861e4e1d8282a267a896eb39d46240fb # v3.35
	with:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	MERGE_COVERAGE_FILES: true
	MINIMUM_GREEN: 90
	MINIMUM_ORANGE: 70

	- name: Store Pull Request comment to be posted
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
	if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
	with:
	name: python-coverage-comment-action
	path: python-coverage-comment-action.txt


	test-mi300x:
	name: "Unit Tests (${{ matrix.runs-on }}, ${{ matrix.python-version }}, ${{ matrix.torch-version }})"
	strategy:
	matrix:
	python-version: ["3.12"]
	torch-version: ["2.5.1"]
	iree-target-args: ["--iree-hip-target=gfx942 --iree-hal-target-device=hip"]
	runs-on: [linux-mi300-8gpu-ossci-nod-ai]
	fail-fast: false
	runs-on: ${{matrix.runs-on}}
	defaults:
	run:
	shell: bash
	env:
	PIP_CACHE_DIR: "${{ github.workspace }}/.pip-cache"
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

	- name: "Setting up Python"
	id: setup_python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: ${{matrix.python-version}}

	- name: Cache Pip Packages
	uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
	id: cache-pip
	with:
	path: ${{ env.PIP_CACHE_DIR }}
	key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt','sharktank/requirements*.txt') }}

	- name: Install pip deps
	run: \|
	python -m pip install --no-compile --upgrade pip

	# Note: We install in three steps in order to satisfy requirements
	# from non default locations first.
	pip install --no-compile \
	--index-url https://download.pytorch.org/whl/rocm6.2 torch==${{matrix.torch-version}}+rocm6.2
	pip install -r requirements-iree-pinned.txt
	pip install --no-compile \
	-r sharktank/requirements-tests.txt \
	-e sharktank/

	pip freeze

	- name: Run sharktank tests
	if: ${{ !cancelled() }}
	run: \|
	pytest sharktank/ \
	--durations=10 \
	--capture=no \
	--log-cli-level=info \
	-v \
	--iree-hal-target-device=hip \
	--iree-hip-target=gfx942 \
	--iree-device=hip://0 \
	--device=cuda:0

	test_with_data:
	name: "Data-dependent Tests"
	strategy:
	matrix:
	python-version: [3.11]
	runs-on: [linux-mi300-1gpu-ossci-nod-ai]
	fail-fast: false
	runs-on: ${{matrix.runs-on}}
	defaults:
	run:
	shell: bash
	env:
	VENV_DIR: ${{ github.workspace }}/.venv
	HF_HOME: "/shark-cache/data/huggingface"
	HF_TOKEN: ${{ secrets.HF_FLUX_TOKEN }}
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

	- name: "Setting up Python"
	id: setup_python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: ${{matrix.python-version}}

	- name: Create Python venv
	run: python -m venv ${VENV_DIR}

	- name: Install sharktank deps
	run: \|
	source ${VENV_DIR}/bin/activate
	python -m pip install --no-compile --upgrade pip

	# Note: We install in three steps in order to satisfy requirements
	# from non default locations first.
	pip install --no-compile -r pytorch-cpu-requirements.txt
	pip install -r requirements-iree-pinned.txt
	pip install --no-compile \
	-r sharktank/requirements-tests.txt \
	-e sharktank/

	pip freeze

	- name: Run tests
	# TODO: unify with-*-data flags into a single flag and make it possible to run
	# only tests that require data.
	# We would still want the separate flags as we may endup with data being
	# scattered on different CI machines.
	run: \|
	source ${VENV_DIR}/bin/activate
	pytest \
	-v \
	--log-cli-level=info \
	--with-quark-data \
	--iree-hal-target-device=hip \
	--iree-hip-target=gfx942 \
	--iree-device=hip://0 \
	sharktank/tests/models/llama/quark_parity_test.py \
	--durations=0 \
	--timeout=800

	test_integration:
	name: "Model Integration Tests"
	runs-on: ubuntu-24.04
	env:
	PIP_CACHE_DIR: "${{ github.workspace }}/.pip-cache"
	steps:
	- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

	- name: "Setting up Python"
	id: setup_python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: 3.11

	- name: Cache Pip Packages
	uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
	id: cache-pip
	with:
	path: ${{ env.PIP_CACHE_DIR }}
	key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt','sharktank/requirements*.txt') }}

	- name: Install pip deps
	run: \|
	python -m pip install --no-compile --upgrade pip

	# Note: We install in three steps in order to satisfy requirements
	# from non default locations first.
	pip install --no-compile -r pytorch-cpu-requirements.txt
	pip install -r requirements-iree-pinned.txt
	pip install --no-compile \
	-r sharktank/requirements-tests.txt \
	-e sharktank/

	pip freeze

	- name: Run punet tests
	run: \|
	pytest -v sharktank/ -m punet_quick \
	--durations=0 \
	--timeout=900

	# Depends on other jobs to provide an aggregate job status.
	# TODO(#584): move test_with_data and test_integration to a pkgci integration test workflow?
	ci_sharktank_summary:
	if: always()
	runs-on: ubuntu-24.04
	needs:
	- test
	steps:
	- name: Getting failed jobs
	run: \|
	echo '${{ toJson(needs) }}'
	FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
	\| jq --raw-output \
	'map_values(select(.result!="success" and .result!="skipped")) \| keys \| join(",")' \
	)"
	if [[ "${FAILED_JOBS}" != "" ]]; then
	echo "The following jobs failed: ${FAILED_JOBS}"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[fusili] Dependency cleanup (remove IREE/LLVM/Catch2 source builds, bring in `lit` & `filecheck` standalone) #7415

Workflow file

[fusili] Dependency cleanup (remove IREE/LLVM/Catch2 source builds, bring in `lit` & `filecheck` standalone) #7415

Uh oh!

Jobs

Run details

Workflow file for this run

[fusili] Dependency cleanup (remove IREE/LLVM/Catch2 source builds, bring in lit & filecheck standalone) #7415

Workflow file

Workflow file for this run

[fusili] Dependency cleanup (remove IREE/LLVM/Catch2 source builds, bring in `lit` & `filecheck` standalone) #7415