Merge bitcoin/bitcoin#33810: ci: Add IWYU job

56750c4 iwyu, clang-format: Sort includes (Hennadii Stepanov)
2c78814 ci: Add IWYU job (Hennadii Stepanov)
94e4f04 cmake: Fix target name (Hennadii Stepanov)
0f81e00 cmake: Make `codegen` target dependent on `generate_build_info` (Hennadii Stepanov)
73f7844 iwyu: Add patch to prefer C++ headers over C counterparts (Hennadii Stepanov)
7a65437 iwyu: Add patch to prefer angled brackets over quotes for includes (Hennadii Stepanov)

Pull request description:

  This PR separates the IWYU checks into its own CI job to provide faster feedback to developers. No other changes are made to the treatment of IWYU warnings. The existing “tidy” CI job will no longer run IWYU.

  See also the discussion of bitcoin/bitcoin#33779, specifically this [comment](bitcoin/bitcoin#33779 (comment)):
  > Maybe a better approach would be to run the enforced sections in a separate, faster job? Some of the linters are already a bit annoying to invoke locally, so I usually just run the lint job. Doing the same for the includes seems fine to me.

  Based on ideas from bitcoin/bitcoin#32953.

ACKs for top commit:
  maflcko:
    review ACK 56750c4 🌄
  sedited:
    ACK 56750c4

Tree-SHA512: af15326b6d0c5d1e11346ac64939644936c65eb9466cd1a17ab5da347d39aef10f7ab33b39fbca31ad291b0b4b54639b147b24410f4f86197e4a776049882694

Author: hebasto

.github/workflows/ci.yml

@@ -522,6 +522,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - name: 'iwyu'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_iwyu.sh'
+
           - name: '32 bit ARM'
             cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode
             fallback-runner: 'ubuntu-24.04-arm'

.gitignore

@@ -12,6 +12,7 @@
 
 # Only ignore unexpected patches
 *.patch
+!ci/test/*.patch
 !contrib/guix/patches/*.patch
 !depends/patches/**/*.patch
 

ci/test/00_setup_env_native_iwyu.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2023-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie"  # To build codegen, CMake must be 3.31 or newer.
+export CONTAINER_NAME=ci_native_iwyu
+export TIDY_LLVM_V="21"
+export APT_LLVM_V="${TIDY_LLVM_V}"
+export PACKAGES="clang-${TIDY_LLVM_V} clang-format-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto"
+export NO_DEPENDS=1
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export RUN_FUZZ_TESTS=false
+export RUN_CHECK_DEPS=false
+export RUN_IWYU=true
+export GOAL="codegen"
+export BITCOIN_CONFIG="\
+ --preset dev-mode -DBUILD_GUI=OFF \
+ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \
+ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \
+"

ci/test/01_base_install.sh

@@ -79,8 +79,9 @@ if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then
   rm -rf /llvm-project
 fi
 
-if [[ "${RUN_TIDY}" == "true" ]]; then
+if [[ "${RUN_IWYU}" == true ]]; then
   ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use
+  (cd /include-what-you-use && patch -p1 < /ci_container_base/ci/test/01_iwyu.patch)
   cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use
   make -C /iwyu-build/ install "$MAKEJOBS"
 fi

ci/test/01_iwyu.patch

@@ -44,10 +44,10 @@ echo "=== BEGIN env ==="
 env
 echo "=== END env ==="
 
-# Don't apply patches in the tidy job, because it relies on the `git diff`
-# command to detect IWYU errors. It is safe to skip this patch in the tidy job
+# Don't apply patches in the iwyu job, because it relies on the `git diff`
+# command to detect IWYU errors. It is safe to skip this patch in the iwyu job
 # because it doesn't run a UB detector.
-if [ "$RUN_TIDY" != "true" ]; then
+if [[ "${RUN_IWYU}" != true ]]; then
   # compact->outputs[i].file_size is uninitialized memory, so reading it is UB.
   # The statistic bytes_written is only used for logging, which is disabled in
   # CI, so as a temporary minimal fix to work around UB and CI failures, leave
@@ -123,7 +123,7 @@ BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST}
 
 BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR -Werror=dev"
 
-if [[ "${RUN_TIDY}" == "true" ]]; then
+if [[ "${RUN_IWYU}" == true || "${RUN_TIDY}" == true ]]; then
   BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
 fi
 
@@ -135,11 +135,15 @@ cmake -S "$BASE_ROOT_DIR" -B "$BASE_BUILD_DIR" "${CMAKE_ARGS[@]}" || (
   false
 )
 
+if [[ "${GOAL}" != all && "${GOAL}" != codegen ]]; then
+  GOAL="all ${GOAL}"
+fi
+
 # shellcheck disable=SC2086
-cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target all $GOAL || (
+cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target $GOAL || (
   echo "Build failure. Verbose build follows."
   # shellcheck disable=SC2086
-  cmake --build "${BASE_BUILD_DIR}" -j1 --target all $GOAL --verbose
+  cmake --build "${BASE_BUILD_DIR}" -j1 --target $GOAL --verbose
   false
 )
 
@@ -207,7 +211,9 @@ if [ "${RUN_TIDY}" = "true" ]; then
     echo "^^^ ⚠️ Failure generated from clang-tidy"
     false
   fi
+fi
 
+if [[ "${RUN_IWYU}" == true ]]; then
   # TODO: Consider enforcing IWYU across the entire codebase.
   FILES_WITH_ENFORCED_IWYU="/src/((crypto|index)/.*\\.cpp|node/blockstorage.cpp|node/utxo_snapshot.cpp|core_read.cpp|signet.cpp|kernel/chain.cpp)"
   jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json"
@@ -223,6 +229,7 @@ if [ "${RUN_TIDY}" = "true" ]; then
              -Xiwyu --max_line_length=160 \
              2>&1 | tee /tmp/iwyu_ci.out
     python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
+    git diff -U0 | ./contrib/devtools/clang-format-diff.py -binary="clang-format-${TIDY_LLVM_V}" -p1 -i -v
   }
 
   run_iwyu "compile_commands_iwyu_errors.json"

ci/test/03_test_script.sh

@@ -16,7 +16,7 @@ ENV BASE_ROOT_DIR=${BASE_ROOT_DIR}
 
 # Make retry available in PATH, needed for CI_RETRY_EXE
 COPY ./ci/retry/retry /usr/bin/retry
-COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/
+COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh ./ci/test/01_iwyu.patch /ci_container_base/ci/test/
 
 # Bash is required, so install it when missing
 RUN sh -c "bash -c 'true' || ( apk update && apk add --no-cache bash )"

ci/test_imagefile

@@ -27,7 +27,7 @@ function(add_libmultiprocess subdir)
   mark_as_advanced(CapnProto_kj-tls_IMPORTED_LOCATION)
   if(BUILD_TESTS)
     # Add tests to "all" target so ctest can run them
-    set_target_properties(mptests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    set_target_properties(mptest PROPERTIES EXCLUDE_FROM_ALL OFF)
   endif()
   # Exclude examples from compilation database, because the examples are not
   # built by default, and they contain generated c++ code. Without this

cmake/libmultiprocess.cmake

@@ -13,4 +13,8 @@
   { "symbol": ["SEEK_CUR", "private", "<cstdio>", "public"] },
   { "symbol": ["SEEK_END", "private", "<cstdio>", "public"] },
   { "symbol": ["SEEK_SET", "private", "<cstdio>", "public"] },
+
+  # IWYU bug.
+  # See: https://github.com/include-what-you-use/include-what-you-use/issues/1863.
+  { "symbol": ["std::vector", "private", "<vector>", "public"] },
 ]

contrib/devtools/iwyu/bitcoin.core.imp

@@ -97,7 +97,7 @@ ForEachMacros:
   - BOOST_FOREACH
 IfMacros:
   - KJ_IF_MAYBE
-IncludeBlocks:   Preserve
+IncludeBlocks:   Regroup
 IncludeCategories:
   - Regex:           '^<bitcoin-build-config\.h>'
     Priority:        -1