Merge branch 'main' into integrates/iree

Author: monorimet

.github/workflows/ci-sharkfuser.yml

@@ -35,7 +35,8 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     defaults:
       run:
-        shell: bash
+        # github actions swaps the {0} at the end for your script
+        shell: bash --noprofile --norc -exo pipefail {0}
     strategy:
       fail-fast: false
       matrix:
@@ -47,6 +48,7 @@ jobs:
               -DCMAKE_C_COMPILER=clang-18
               -DCMAKE_CXX_COMPILER=clang++-18
               -DCMAKE_LINKER_TYPE=LLD
+              -DPython3_EXECUTABLE=$(which python3)
               -DSHARKFUSER_DEBUG_BUILD=OFF
               -DSHARKFUSER_CODE_COVERAGE=OFF
             additional-packages: clang lld
@@ -59,6 +61,7 @@ jobs:
               -DCMAKE_C_COMPILER=clang-18
               -DCMAKE_CXX_COMPILER=clang++-18
               -DCMAKE_LINKER_TYPE=LLD
+              -DPython3_EXECUTABLE=$(which python3)
               -DSHARKFUSER_DEBUG_BUILD=ON
               -DSHARKFUSER_CODE_COVERAGE=OFF
             additional-packages: clang lld
@@ -70,6 +73,7 @@ jobs:
             cmake-options:
               -DCMAKE_C_COMPILER=gcc-13
               -DCMAKE_CXX_COMPILER=g++-13
+              -DPython3_EXECUTABLE=$(which python3)
               -DSHARKFUSER_DEBUG_BUILD=OFF
               -DSHARKFUSER_CODE_COVERAGE=ON
             additional-packages: lcov

docs/shortfin/llm/user/llama_serving.md

@@ -385,7 +385,8 @@ python -m sharktank.examples.export_paged_llm_v1 \
   --output-config /path/to/output/llama3.1-405b.config.json \
   --bs-prefill 4 \
   --bs-decode 4 \
-  --use-attention-mask
+  --use-attention-mask \
+  --tensor-parallelism-size=8
 ```
 
 ### Compiling to VMFB

sharkfuser/CMakeLists.txt

@@ -77,11 +77,14 @@ message(STATUS "  - Host")
 ################################################################################
 
 set(IREE_VISIBILITY_HIDDEN OFF)
-set(IREE_BUILD_COMPILER OFF)
+set(IREE_BUILD_COMPILER ON)
 set(IREE_BUILD_TESTS OFF)
 set(IREE_BUILD_SAMPLES OFF)
 set(IREE_ERROR_ON_MISSING_SUBMODULES OFF)
 set(IREE_HAL_DRIVER_DEFAULTS OFF)
+set(IREE_TARGET_BACKEND_DEFAULTS OFF)
+set(IREE_INPUT_STABLEHLO OFF)
+set(IREE_INPUT_TOSA OFF)
 set(IREE_HAL_DRIVER_LOCAL_SYNC OFF)
 set(IREE_HAL_DRIVER_LOCAL_TASK OFF)
 set(IREE_HAL_DRIVER_VULKAN OFF)
@@ -94,7 +97,15 @@ if(SHARKFUSER_IREE_SOURCE_DIR)
   add_subdirectory(${SHARKFUSER_IREE_SOURCE_DIR} sharkfuser_iree SYSTEM EXCLUDE_FROM_ALL)
 else()
   message(STATUS "Fetching IREE sources from tag ${SHARKFUSER_IREE_GIT_TAG}")
-  set(IREE_SUBMODULES "third_party/benchmark third_party/cpuinfo third_party/flatcc third_party/hip-build-deps")
+  set(IREE_SUBMODULES "")
+  list(APPEND IREE_SUBMODULES "third_party/benchmark")
+  list(APPEND IREE_SUBMODULES "third_party/cpuinfo")
+  list(APPEND IREE_SUBMODULES "third_party/flatcc")
+  list(APPEND IREE_SUBMODULES "third_party/hip-build-deps")
+  if(IREE_BUILD_COMPILER)
+    list(APPEND IREE_SUBMODULES "third_party/llvm-project")
+    list(APPEND IREE_SUBMODULES "third_party/torch-mlir")
+  endif()
   FetchContent_Declare(
     sharkfuser_iree
     GIT_REPOSITORY  https://github.com/iree-org/iree.git
@@ -107,6 +118,18 @@ else()
   FetchContent_GetProperties(sharkfuser_iree)
   if(NOT sharkfuser_iree_POPULATED)
     FetchContent_MakeAvailable(sharkfuser_iree)
+
+    # LLVM repo has several .gcda and .gcno files as test fixtures for llvm-cov.
+    # Those .g* files will have version issues + errors when importing with our
+    # code coverage. It's remarkably difficult to convince lcov to ignore files,
+    # so the easiest thing to do is just strip them out.
+    message(STATUS "Removing codecov files from LLVM")
+    execute_process(
+      COMMAND ${CMAKE_COMMAND}
+              -DSHARKFUSER_IREE_SOURCE_DIR=${sharkfuser_iree_SOURCE_DIR}
+              -P ${CMAKE_CURRENT_SOURCE_DIR}/build_tools/cmake/strip_codecov_files.cmake
+      WORKING_DIRECTORY ${sharkfuser_iree_SOURCE_DIR}
+    )
   endif()
 endif()
 
@@ -119,6 +142,15 @@ endif()
 # Build tests
 if(SHARKFUSER_BUILD_TESTS)
   message(STATUS "Building SharkFuser tests")
+
+  # Require Python 3.8+ for lit tests
+  find_package(Python3 3.8 REQUIRED COMPONENTS Interpreter)
+
+  # Grab lit from IREE LLVM. We could also look into using a python pip
+  # dependency.
+  set(LLVM_EXTERNAL_LIT "${LLVM_SOURCE_DIR}/utils/lit/lit.py")
+  message(STATUS "Using LIT from IREE LLVM: ${LLVM_EXTERNAL_LIT}")
+
   add_subdirectory(tests)
   enable_testing()
 endif()

sharkfuser/README.md

@@ -11,11 +11,17 @@ A side note on naming: 'SharkFuser' is the name of the project (may change as th
 ## Developer Guide:
 
 ### Build and test (debug build):
+
+**Python Requirements:** [`lit`](https://llvm.org/docs/CommandGuide/lit.html)
+tests require Python. No external Python dependencies are needed - you can use
+your system Python or create a virtual environment.
+
 ```shell
 cmake -GNinja -S. -Bbuild \
     -DCMAKE_C_COMPILER=clang \
     -DCMAKE_CXX_COMPILER=clang++ \
     -DCMAKE_LINKER_TYPE=LLD \
+    -DPython3_EXECUTABLE=$(which python3) \
     -DSHARKFUSER_DEBUG_BUILD=ON
 cmake --build build --target all
 ctest --test-dir build
@@ -37,6 +43,7 @@ To generate code coverage metrics:
 cmake -GNinja -S. -Bbuild \
     -DCMAKE_C_COMPILER=gcc \
     -DCMAKE_CXX_COMPILER=g++ \
+    -DPython3_EXECUTABLE=$(which python3) \
     -DSHARKFUSER_CODE_COVERAGE=ON
 cmake --build build --target all
 ctest --test-dir build -T test -T coverage

sharkfuser/build_tools/cmake/CTestMacros.cmake

@@ -5,7 +5,21 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 
-function(_add_sharkfuser_target)
+# Creates an executable target for use in a test.
+#
+# NAME
+#  The name of the executable target to create (required)
+#
+# SRCS
+#  Source files to compile into the executable (required)
+#
+# DEPS
+#  Additional library dependencies beyond the standard ones
+#  (libfusili and Catch2::Catch2WithMain are always linked)
+#
+# BIN_SUBDIR
+#  Subdirectory under build/bin/ where the executable will be placed
+function(_add_sharkfuser_executable_for_test)
   cmake_parse_arguments(
     _RULE               # prefix
     ""                  # options
@@ -29,6 +43,33 @@ function(_add_sharkfuser_target)
     target_link_options(${_RULE_NAME} PRIVATE -coverage)
   endif()
 
+  # Place executable in the build/bin sub-directory
+  set_target_properties(
+      ${_RULE_NAME} PROPERTIES
+      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/${_RULE_BIN_SUBDIR}
+  )
+endfunction()
+
+
+# Creates an executable target + test.
+function(_add_sharkfuser_ctest_target)
+  cmake_parse_arguments(
+    _RULE               # prefix
+    ""                  # options
+    "NAME;BIN_SUBDIR"   # one value keywords
+    "SRCS;DEPS"         # multi-value keywords
+    ${ARGN}             # extra arguments
+  )
+
+  # Create the target first
+  _add_sharkfuser_executable_for_test(
+    NAME ${_RULE_NAME}
+    SRCS ${_RULE_SRCS}
+    DEPS ${_RULE_DEPS}
+    BIN_SUBDIR ${_RULE_BIN_SUBDIR}
+  )
+
+  # Add the test
   add_test(NAME ${_RULE_NAME} COMMAND ${_RULE_NAME})
 
   # Set logging environment variables
@@ -38,16 +79,25 @@ function(_add_sharkfuser_target)
       ENVIRONMENT "FUSILI_LOG_INFO=1;FUSILI_LOG_FILE=stdout"
     )
   endif()
-
-  # Place executable in the build/bin sub-directory
-  set_target_properties(
-      ${_RULE_NAME} PROPERTIES
-      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/${_RULE_BIN_SUBDIR}
-  )
 endfunction()
 
 
+# Creates a sharkfuser test.
+#
+# NAME
+#  The name of the executable target to create (required)
+#
+# SRCS
+#  Source files to compile into the executable (required)
+#
+# DEPS
+#  Additional library dependencies beyond the standard ones
+#  (libfusili and Catch2::Catch2WithMain are always linked)
 function(add_sharkfuser_test)
+  if(NOT SHARKFUSER_BUILD_TESTS)
+    return()
+  endif()
+
   cmake_parse_arguments(
     _RULE
     ""
@@ -56,11 +106,7 @@ function(add_sharkfuser_test)
     ${ARGN}
   )
 
-  if(NOT SHARKFUSER_BUILD_TESTS)
-    return()
-  endif()
-
-  _add_sharkfuser_target(
+  _add_sharkfuser_ctest_target(
     NAME ${_RULE_NAME}
     SRCS ${_RULE_SRCS}
     DEPS ${_RULE_DEPS}
@@ -69,7 +115,22 @@ function(add_sharkfuser_test)
 endfunction()
 
 
+# Creates a sharkfuser sample.
+#
+# NAME
+#  The name of the executable target to create (required)
+#
+# SRCS
+#  Source files to compile into the executable (required)
+#
+# DEPS
+#  Additional library dependencies beyond the standard ones
+#  (libfusili and Catch2::Catch2WithMain are always linked)
 function(add_sharkfuser_sample)
+  if(NOT SHARKFUSER_BUILD_SAMPLES)
+    return()
+  endif()
+
   cmake_parse_arguments(
     _RULE
     ""
@@ -78,14 +139,87 @@ function(add_sharkfuser_sample)
     ${ARGN}
   )
 
-  if(NOT SHARKFUSER_BUILD_SAMPLES)
-    return()
-  endif()
-
-  _add_sharkfuser_target(
+  _add_sharkfuser_ctest_target(
     NAME ${_RULE_NAME}
     SRCS ${_RULE_SRCS}
     DEPS ${_RULE_DEPS}
     BIN_SUBDIR samples
   )
 endfunction()
+
+
+# Creates a lit test that compiles a source file and runs lit on it.
+#
+#  add_sharkfuser_lit_test(
+#    SRC <source-file>
+#    [TOOLS <tool1> <tool2> ...]
+#    [DEPS <dep1> <dep2> ...]
+#  )
+#
+# SRC
+#  The source file to compile and test (required)
+#
+# TOOLS
+#  External tools needed for the test (e.g., FileCheck)
+#
+# DEPS
+#  Library dependencies for the compiled executable
+function(add_sharkfuser_lit_test)
+  if(NOT SHARKFUSER_BUILD_TESTS)
+    return()
+  endif()
+
+  cmake_parse_arguments(
+    _RULE               # prefix
+    ""                  # options
+    "SRC"               # one value keywords
+    "TOOLS;DEPS"        # multi-value keywords
+    ${ARGN}
+  )
+  if(NOT _RULE_SRC)
+    message(FATAL_ERROR "add_sharkfuser_lit_test: SRC parameter is required")
+  endif()
+
+  get_filename_component(_ABSOLUTE_RULE_SRC ${_RULE_SRC} ABSOLUTE)
+  get_filename_component(_TEST_NAME ${_RULE_SRC} NAME_WE)
+
+  # The executable who's output is being lit tested.
+  _add_sharkfuser_executable_for_test(
+    NAME ${_TEST_NAME}
+    SRCS ${_RULE_SRC}
+    DEPS ${_RULE_DEPS}
+    BIN_SUBDIR lit
+  )
+
+  # Pass lit locations of tools in build directory through `--path` arguments.
+  set(_LIT_PATH_ARGS
+    "--path" "$<TARGET_FILE_DIR:${_TEST_NAME}>" # include test itself
+    "--path" "$<TARGET_FILE_DIR:FileCheck>"     # include FileCheck by default
+    )
+  foreach(_TOOL IN LISTS _RULE_TOOLS)
+    list(APPEND _LIT_PATH_ARGS "--path" "$<TARGET_FILE_DIR:${_TOOL}>")
+  endforeach()
+
+  add_test(
+    NAME
+      ${_TEST_NAME}
+    COMMAND
+      "${Python3_EXECUTABLE}"
+      "${LLVM_EXTERNAL_LIT}"
+      "${_ABSOLUTE_RULE_SRC}"
+      ${_LIT_PATH_ARGS}
+      # lit config sets the "%test_exe" substitution based on this param.
+      "--param" "TEST_EXE=$<TARGET_FILE:${_TEST_NAME}>"
+      # Ensures lit prints a (more) useful error message on failure.
+      "--verbose"
+  )
+
+  # Apparently this flag helps FileCheck spit out nicer error messages.
+  set_tests_properties(${_TEST_NAME} PROPERTIES ENVIRONMENT
+    "FILECHECK_OPTS=--enable-var-scope")
+
+  # Dependencies for the test.
+  if(_RULE_TOOLS)
+    set_tests_properties(${_TEST_NAME} PROPERTIES DEPENDS "${_RULE_TOOLS}")
+  endif()
+endfunction()

sharkfuser/build_tools/cmake/strip_codecov_files.cmake

@@ -0,0 +1,23 @@
+if(NOT DEFINED SHARKFUSER_IREE_SOURCE_DIR)
+  message(FATAL_ERROR "SHARKFUSER_IREE_SOURCE_DIR must be defined")
+endif()
+
+set(THIRD_PARTY_DIR "${SHARKFUSER_IREE_SOURCE_DIR}/third_party")
+
+# Find all .gcda files
+file(GLOB_RECURSE GCDA_FILES "${THIRD_PARTY_DIR}/*.gcda")
+
+# Find all .gcno files
+file(GLOB_RECURSE GCNO_FILES "${THIRD_PARTY_DIR}/*.gcno")
+
+# Combine the lists
+set(CODECOV_FILES ${GCDA_FILES} ${GCNO_FILES})
+
+message(STATUS "CODECOV_FILES: ${CODECOV_FILES}")
+
+# Remove files
+if(CODECOV_FILES)
+  file(REMOVE ${CODECOV_FILES})
+else()
+  message(STATUS "No .gcda/.gcno files found.")
+endif()