Improve C++ test infrastructure and disable hanging test

This commit improves the C++ test infrastructure to ensure test output
is visible in CI logs, and disables a test that hangs on free-threaded
Python 3.14+.

Changes:

## CI/test infrastructure improvements

- .github/workflows: Added `timeout-minutes: 3` to all C++ test steps
  to prevent indefinite hangs.

- tests/**/CMakeLists.txt: Added `USES_TERMINAL` to C++ test targets
  (cpptest, test_cross_module_rtti, test_pure_cpp) to ensure output is
  shown immediately rather than buffered and possibly lost on crash/timeout.

- tests/test_with_catch/catch.cpp: Added a custom Catch2 progress reporter
  with timestamps, Python version info, and a SIGTERM handler to make test
  execution and failures clearly visible in CI logs.

## Disabled hanging test

- The "Move Subinterpreter" test is disabled on free-threaded Python 3.14+
  due to a hang in Py_EndInterpreter() when the subinterpreter is destroyed
  from a different thread than it was created on. Work on fixing the
  underlying issue will continue under PR pybind#5940.

Context: We were in the dark for months (since we started testing with
Python 3.14t) because CI logs gave no clue about the root cause of hangs.
This led to ignoring intermittent hangs (mostly on macOS). Our hand was
forced only with the Python 3.14.1 release, when hangs became predictable
on all platforms.

For the full development history of these changes, see PR pybind#5933.

Author: rwgk

.github/workflows/ci.yml

@@ -229,6 +229,7 @@ jobs:
       run: cmake --build . --target pytest
 
     - name: Compiled tests
+      timeout-minutes: 3
       run: cmake --build . --target cpptest
 
     - name: Interface test
@@ -334,6 +335,7 @@ jobs:
       run: cmake --build --preset default --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build --preset default --target cpptest
 
     - name: Visibility test
@@ -393,6 +395,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -516,6 +519,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -570,6 +574,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -652,6 +657,7 @@ jobs:
         cmake --build build-11 --target check
 
     - name: C++ tests C++11
+      timeout-minutes: 3
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
         cmake --build build-11 --target cpptest
@@ -689,6 +695,7 @@ jobs:
         cmake --build build-17 --target check
 
     - name: C++ tests C++17
+      timeout-minutes: 3
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
         cmake --build build-17 --target cpptest
@@ -760,6 +767,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++ tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest
 
     - name: Interface test
@@ -1000,6 +1008,7 @@ jobs:
       run: cmake --build build --target pytest
 
     - name: C++20 tests
+      timeout-minutes: 3
       run: cmake --build build --target cpptest -j 2
 
     - name: Interface test C++20
@@ -1076,6 +1085,7 @@ jobs:
       run: cmake --build build --target pytest -j 2
 
     - name: C++11 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target cpptest -j 2
 
     - name: Interface test C++11
@@ -1100,6 +1110,7 @@ jobs:
       run: cmake --build build2 --target pytest -j 2
 
     - name: C++14 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target cpptest -j 2
 
     - name: Interface test C++14
@@ -1124,6 +1135,7 @@ jobs:
       run: cmake --build build3 --target pytest -j 2
 
     - name: C++17 tests
+      timeout-minutes: 3
       run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target cpptest -j 2
 
     - name: Interface test C++17
@@ -1195,6 +1207,7 @@ jobs:
         run: cmake --build . --target pytest -j 2
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build . --target cpptest -j 2
 
       - name: Interface test
@@ -1257,6 +1270,7 @@ jobs:
         run: cmake --build . --target pytest -j 2
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build . --target cpptest -j 2
 
       - name: Interface test
@@ -1329,6 +1343,7 @@ jobs:
       run: cmake --build build --target pytest -j 2
 
     - name: C++ tests
+      timeout-minutes: 3
       run: PYTHONHOME=/clangarm64 PYTHONPATH=/clangarm64 cmake --build build --target cpptest -j 2
 
     - name: Interface test

.github/workflows/reusable-standard.yml

@@ -83,6 +83,7 @@ jobs:
         run: cmake --build build --target pytest
 
       - name: C++ tests
+        timeout-minutes: 3
         run: cmake --build build --target cpptest
 
       - name: Interface test

.github/workflows/upstream.yml

@@ -66,6 +66,7 @@ jobs:
       run: cmake --build build11 --target pytest -j 2
 
     - name: C++11 tests
+      timeout-minutes: 3
       run: cmake --build build11  --target cpptest -j 2
 
     - name: Interface test C++11
@@ -87,6 +88,7 @@ jobs:
       run: cmake --build build17 --target pytest
 
     - name: C++17 tests
+      timeout-minutes: 3
       run: cmake --build build17 --target cpptest
 
     # Third build - C++17 mode with unstable ABI

tests/pure_cpp/CMakeLists.txt

@@ -15,6 +15,8 @@ target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Ca
 add_custom_target(
   test_pure_cpp
   COMMAND "$<TARGET_FILE:smart_holder_poc_test>"
-  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 add_dependencies(check test_pure_cpp)

tests/test_cross_module_rtti/CMakeLists.txt

@@ -60,7 +60,9 @@ add_custom_target(
   test_cross_module_rtti
   COMMAND "$<TARGET_FILE:test_cross_module_rtti_main>"
   DEPENDS test_cross_module_rtti_main
-  WORKING_DIRECTORY "$<TARGET_FILE_DIR:test_cross_module_rtti_main>")
+  WORKING_DIRECTORY "$<TARGET_FILE_DIR:test_cross_module_rtti_main>"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 set_target_properties(test_cross_module_rtti_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                                                                  "${CMAKE_CURRENT_BINARY_DIR}")

tests/test_with_catch/CMakeLists.txt

@@ -47,7 +47,9 @@ add_custom_target(
   cpptest
   COMMAND "$<TARGET_FILE:test_with_catch>"
   DEPENDS test_with_catch
-  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash)
+)
 
 pybind11_add_module(external_module THIN_LTO external_module.cpp)
 set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY

tests/test_with_catch/catch.cpp

@@ -3,6 +3,17 @@
 
 #include <pybind11/embed.h>
 
+#include <chrono>
+#include <csignal>
+#include <cstring>
+#include <ctime>
+#include <iomanip>
+#include <sstream>
+
+#ifndef _WIN32
+#    include <unistd.h>
+#endif
+
 // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to
 // catch 2.0.1; this should be fixed in the next catch release after 2.0.1).
 PYBIND11_WARNING_DISABLE_MSVC(4996)
@@ -13,11 +24,109 @@ PYBIND11_WARNING_DISABLE_MSVC(4996)
 #endif
 
 #define CATCH_CONFIG_RUNNER
+#define CATCH_CONFIG_DEFAULT_REPORTER "progress"
 #include <catch.hpp>
 
 namespace py = pybind11;
 
+// Simple progress reporter that prints a line per test case.
+namespace {
+
+class ProgressReporter : public Catch::StreamingReporterBase<ProgressReporter> {
+public:
+    using StreamingReporterBase<ProgressReporter>::StreamingReporterBase;
+
+    static std::string getDescription() { return "Simple progress reporter (one line per test)"; }
+
+    void testCaseStarting(Catch::TestCaseInfo const &testInfo) override {
+        print_python_version_once();
+        auto &os = Catch::cout();
+        os << "[ RUN      ] " << testInfo.name << '\n';
+        os.flush();
+    }
+
+    void testCaseEnded(Catch::TestCaseStats const &stats) override {
+        bool failed = stats.totals.assertions.failed > 0;
+        auto &os = Catch::cout();
+        os << (failed ? "[  FAILED  ] " : "[       OK ] ") << stats.testInfo.name << '\n';
+        os.flush();
+    }
+
+    void noMatchingTestCases(std::string const &spec) override {
+        auto &os = Catch::cout();
+        os << "[  NO TEST ] no matching test cases for spec: " << spec << '\n';
+        os.flush();
+    }
+
+    void reportInvalidArguments(std::string const &arg) override {
+        auto &os = Catch::cout();
+        os << "[   ERROR  ] invalid Catch2 arguments: " << arg << '\n';
+        os.flush();
+    }
+
+    void assertionStarting(Catch::AssertionInfo const &) override {}
+
+    bool assertionEnded(Catch::AssertionStats const &) override { return false; }
+
+private:
+    void print_python_version_once() {
+        if (printed_) {
+            return;
+        }
+        printed_ = true;
+        auto &os = Catch::cout();
+        os << "[ PYTHON   ] " << Py_GetVersion() << '\n';
+        os.flush();
+    }
+
+    bool printed_ = false;
+};
+
+} // namespace
+
+CATCH_REGISTER_REPORTER("progress", ProgressReporter)
+
+namespace {
+
+std::string get_utc_timestamp() {
+    auto now = std::chrono::system_clock::now();
+    auto time_t_now = std::chrono::system_clock::to_time_t(now);
+    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()) % 1000;
+
+    std::tm utc_tm{};
+#if defined(_WIN32)
+    gmtime_s(&utc_tm, &time_t_now);
+#else
+    gmtime_r(&time_t_now, &utc_tm);
+#endif
+
+    std::ostringstream oss;
+    oss << std::put_time(&utc_tm, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') << std::setw(3)
+        << ms.count() << 'Z';
+    return oss.str();
+}
+
+#ifndef _WIN32
+// Signal handler to print a message when the process is terminated.
+// Uses only async-signal-safe functions.
+void termination_signal_handler(int sig) {
+    const char *msg = "[  SIGNAL  ] Process received SIGTERM\n";
+    // write() is async-signal-safe, unlike std::cout
+    ssize_t written = write(STDOUT_FILENO, msg, strlen(msg));
+    (void) written; // suppress "unused variable" warnings
+    // Re-raise with default handler to get proper exit status
+    std::signal(sig, SIG_DFL);
+    std::raise(sig);
+}
+#endif
+
+} // namespace
+
 int main(int argc, char *argv[]) {
+#ifndef _WIN32
+    std::signal(SIGTERM, termination_signal_handler);
+#endif
+
     // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number:
     std::string updated_pythonpath("pybind11_test_with_catch_PYTHONPATH_2099743835476552");
     const char *preexisting_pythonpath = getenv("PYTHONPATH");
@@ -35,9 +144,15 @@ int main(int argc, char *argv[]) {
     setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1);
 #endif
 
+    std::cout << "[ STARTING ] " << get_utc_timestamp() << '\n';
+    std::cout.flush();
+
     py::scoped_interpreter guard{};
 
     auto result = Catch::Session().run(argc, argv);
 
+    std::cout << "[ DONE     ] " << get_utc_timestamp() << " (result " << result << ")\n";
+    std::cout.flush();
+
     return result < 0xff ? result : 0xff;
 }

tests/test_with_catch/test_subinterpreter.cpp

@@ -90,7 +90,10 @@ TEST_CASE("Single Subinterpreter") {
     unsafe_reset_internals_for_single_interpreter();
 }
 
-#    if PY_VERSION_HEX >= 0x030D0000
+// "Move Subinterpreter" test is disabled on free-threaded Python 3.14+ due to a hang
+// in Py_EndInterpreter() when the subinterpreter is destroyed from a different thread
+// than it was created on. See: https://github.com/pybind/pybind11/pull/5940
+#    if PY_VERSION_HEX >= 0x030D0000 && !(PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED))
 TEST_CASE("Move Subinterpreter") {
     std::unique_ptr<py::subinterpreter> sub(new py::subinterpreter(py::subinterpreter::create()));