Add per-interpreter storage for gil_safe_call_once_and_store

include/pybind11/detail/class.h

@@ -226,14 +226,12 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) {
                 local_internals.registered_types_cpp.erase(tinfo->cpptype);
             } else {
                 internals.registered_types_cpp.erase(tindex);
-#if PYBIND11_INTERNALS_VERSION >= 12
                 internals.registered_types_cpp_fast.erase(tinfo->cpptype);
                 for (const std::type_info *alias : tinfo->alias_chain) {
                     auto num_erased = internals.registered_types_cpp_fast.erase(alias);
                     (void) num_erased;
                     assert(num_erased > 0);
                 }
-#endif
             }
             internals.registered_types_py.erase(tinfo->type);
 

include/pybind11/detail/internals.h

@@ -39,11 +39,11 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    define PYBIND11_INTERNALS_VERSION 11
+#    define PYBIND11_INTERNALS_VERSION 12
 #endif
 
-#if PYBIND11_INTERNALS_VERSION < 11
-#    error "PYBIND11_INTERNALS_VERSION 11 is the minimum for all platforms for pybind11v3."
+#if PYBIND11_INTERNALS_VERSION < 12
+#    error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11v3."
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -234,6 +234,38 @@ inline uint64_t round_up_to_next_pow2(uint64_t x) {
 
 class loader_life_support;
 
+struct call_once_storage_base {
+    call_once_storage_base() = default;
+    virtual ~call_once_storage_base() = default;
+    call_once_storage_base(const call_once_storage_base &) = delete;
+    call_once_storage_base(call_once_storage_base &&) = delete;
+    call_once_storage_base &operator=(const call_once_storage_base &) = delete;
+    call_once_storage_base &operator=(call_once_storage_base &&) = delete;
+};
+
+template <typename T>
+struct call_once_storage : call_once_storage_base {
+    alignas(T) char storage[sizeof(T)] = {};
+    std::once_flag once_flag;
+    void (*finalize)(T &) = nullptr;
+    std::atomic_bool is_initialized{false};
+
+    call_once_storage() = default;
+    ~call_once_storage() override {
+        if (is_initialized) {
+            if (finalize != nullptr) {
+                finalize(*reinterpret_cast<T *>(storage));
+            } else {
+                reinterpret_cast<T *>(storage)->~T();
+            }
+        }
+    };
+    call_once_storage(const call_once_storage &) = delete;
+    call_once_storage(call_once_storage &&) = delete;
+    call_once_storage &operator=(const call_once_storage &) = delete;
+    call_once_storage &operator=(call_once_storage &&) = delete;
+};
+
 /// Internal data structure used to track registered instances and types.
 /// Whenever binary incompatible changes are made to this structure,
 /// `PYBIND11_INTERNALS_VERSION` must be incremented.
@@ -242,14 +274,12 @@ struct internals {
     pymutex mutex;
     pymutex exception_translator_mutex;
 #endif
-#if PYBIND11_INTERNALS_VERSION >= 12
     // non-normative but fast "hint" for registered_types_cpp. Meant
     // to be used as the first level of a two-level lookup: successful
     // lookups are correct, but unsuccessful lookups need to try
     // registered_types_cpp and then backfill this map if they find
     // anything.
     fast_type_map<type_info *> registered_types_cpp_fast;
-#endif
 
     // std::type_index -> pybind11's type information
     type_map<type_info *> registered_types_cpp;
@@ -275,14 +305,13 @@ struct internals {
     PyObject *instance_base = nullptr;
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     thread_specific_storage<PyThreadState> tstate;
-#if PYBIND11_INTERNALS_VERSION <= 11
-    thread_specific_storage<loader_life_support> loader_life_support_tls; // OBSOLETE (PR #5830)
-#endif
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PyInterpreterState *istate = nullptr;
 
     type_map<PyObject *> native_enum_type_map;
 
+    std::unordered_map<const void *, call_once_storage_base *> call_once_storage_map;
+
     internals()
         : static_property_type(make_static_property_type()),
           default_metaclass(make_default_metaclass()) {
@@ -308,7 +337,12 @@ struct internals {
     internals(internals &&other) = delete;
     internals &operator=(const internals &other) = delete;
     internals &operator=(internals &&other) = delete;
-    ~internals() = default;
+    ~internals() {
+        for (const auto &entry : call_once_storage_map) {
+            delete entry.second;
+        }
+        call_once_storage_map.clear();
+    }
 };
 
 // the internals struct (above) is shared between all the modules. local_internals are only
@@ -358,7 +392,6 @@ struct type_info {
     void *(*module_local_load)(PyObject *, const type_info *) = nullptr;
     holder_enum_t holder_enum_v = holder_enum_t::undefined;
 
-#if PYBIND11_INTERNALS_VERSION >= 12
     // When a type appears in multiple DSOs,
     // internals::registered_types_cpp_fast will have multiple distinct
     // keys (the std::type_info from each DSO) mapped to the same
@@ -369,7 +402,6 @@ struct type_info {
     // nb_alias_chain` added in
     // https://github.com/wjakob/nanobind/commit/b515b1f7f2f4ecc0357818e6201c94a9f4cbfdc2
     std::forward_list<const std::type_info *> alias_chain;
-#endif
 
     /* A simple type never occurs as a (direct or indirect) parent
      * of a class that makes use of multiple inheritance.
@@ -564,6 +596,15 @@ class internals_pp_manager {
     /// acquire the GIL. Will never return nullptr.
     std::unique_ptr<InternalsType> *get_pp() {
 #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
+        // FIXME: We cannot use `get_num_interpreters_seen() > 1` here to create a fast path for
+        // the multi-interpreter case. The singleton may be initialized by a subinterpreter not the
+        // main interpreter.
+        //
+        // For multi-interpreter support, the subinterpreters can be initialized concurrently, and
+        // the first time this function may not be called in the main interpreter.
+        // For example, a clean main interpreter that does not import any pybind11 module and then
+        // spawns multiple subinterpreters using `InterpreterPoolExecutor` that each imports a
+        // pybind11 module concurrently.
         if (get_num_interpreters_seen() > 1) {
             // Whenever the interpreter changes on the current thread we need to invalidate the
             // internals_pp so that it can be pulled from the interpreter's state dict.  That is
@@ -580,9 +621,19 @@ class internals_pp_manager {
             return internals_p_tls();
         }
 #endif
-        if (!internals_singleton_pp_) {
-            gil_scoped_acquire_simple gil;
-            internals_singleton_pp_ = get_or_create_pp_in_state_dict();
+        return get_pp_for_main_interpreter();
+    }
+
+    /// Get the pointer-to-pointer for the main interpreter, allocating it if it does not already
+    /// exist.  May acquire the GIL. Will never return nullptr.
+    std::unique_ptr<InternalsType> *get_pp_for_main_interpreter() {
+        // This function **assumes** that the current thread is running in the main interpreter.
+        if (!seen_main_interpreter_) {
+            std::call_once(seen_main_interpreter_flag_, [&] {
+                gil_scoped_acquire_simple gil;
+                internals_singleton_pp_ = get_or_create_pp_in_state_dict();
+                seen_main_interpreter_ = true;
+            });
         }
         return internals_singleton_pp_;
     }
@@ -661,6 +712,9 @@ class internals_pp_manager {
     char const *holder_id_ = nullptr;
     on_fetch_function *on_fetch_ = nullptr;
     std::unique_ptr<InternalsType> *internals_singleton_pp_;
+
+    std::once_flag seen_main_interpreter_flag_;
+    std::atomic_bool seen_main_interpreter_{false};
 };
 
 // If We loaded the internals through `state_dict`, our `error_already_set`

include/pybind11/detail/type_caster_base.h

@@ -227,32 +227,26 @@ inline detail::type_info *get_global_type_info_lock_held(const std::type_info &t
     // next time.
     detail::type_info *type_info = nullptr;
     auto &internals = get_internals();
-#if PYBIND11_INTERNALS_VERSION >= 12
     auto &fast_types = internals.registered_types_cpp_fast;
-#endif
     auto &types = internals.registered_types_cpp;
-#if PYBIND11_INTERNALS_VERSION >= 12
     auto fast_it = fast_types.find(&tp);
     if (fast_it != fast_types.end()) {
-#    ifndef NDEBUG
+#ifndef NDEBUG
         auto types_it = types.find(std::type_index(tp));
         assert(types_it != types.end());
         assert(types_it->second == fast_it->second);
-#    endif
+#endif
         return fast_it->second;
     }
-#endif // PYBIND11_INTERNALS_VERSION >= 12
 
     auto it = types.find(std::type_index(tp));
     if (it != types.end()) {
-#if PYBIND11_INTERNALS_VERSION >= 12
         // We found the type in the slow map but not the fast one, so
         // some other DSO added it (otherwise it would be in the fast
         // map under &tp) and therefore we must be an alias. Record
         // that.
         it->second->alias_chain.push_front(&tp);
         fast_types.emplace(&tp, it->second);
-#endif
         type_info = it->second;
     }
     return type_info;

include/pybind11/gil_safe_call_once.h

@@ -3,13 +3,18 @@
 #pragma once
 
 #include "detail/common.h"
+#include "detail/internals.h"
 #include "gil.h"
 
 #include <cassert>
 #include <mutex>
 
-#ifdef Py_GIL_DISABLED
+#if defined(Py_GIL_DISABLED) || defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT)
 #    include <atomic>
+
+using atomic_bool = std::atomic_bool;
+#else
+using atomic_bool = bool;
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -48,12 +53,19 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 // functions, which is usually the case.
 //
 // For in-depth background, see docs/advanced/deadlock.md
+#ifndef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
+// Subinterpreter support is disabled.
+// In this case, we can store the result globally, because there is only a single interpreter.
+//
+// The life span of the stored result is the entire process lifetime. It is leaked on process
+// termination to avoid destructor calls after the Python interpreter was finalized.
 template <typename T>
 class gil_safe_call_once_and_store {
 public:
     // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called.
     template <typename Callable>
-    gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) {
+    gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn,
+                                                             void (*)(T &) /*unused*/ = nullptr) {
         if (!is_initialized_) { // This read is guarded by the GIL.
             // Multiple threads may enter here, because the GIL is released in the next line and
             // CPython API calls in the `fn()` call below may release and reacquire the GIL.
@@ -74,29 +86,121 @@ class gil_safe_call_once_and_store {
     T &get_stored() {
         assert(is_initialized_);
         PYBIND11_WARNING_PUSH
-#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5
+#    if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5
         // Needed for gcc 4.8.5
         PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing")
-#endif
+#    endif
         return *reinterpret_cast<T *>(storage_);
         PYBIND11_WARNING_POP
     }
 
     constexpr gil_safe_call_once_and_store() = default;
+    // The instance is a global static, so its destructor runs when the process
+    // is terminating. Therefore, do nothing here because the Python interpreter
+    // may have been finalized already.
     PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default;
 
 private:
+    // Global static storage (per process) when subinterpreter support is disabled.
     alignas(T) char storage_[sizeof(T)] = {};
     std::once_flag once_flag_;
-#ifdef Py_GIL_DISABLED
-    std::atomic_bool
-#else
-    bool
-#endif
-        is_initialized_{false};
+
     // The `is_initialized_`-`storage_` pair is very similar to `std::optional`,
     // but the latter does not have the triviality properties of former,
     // therefore `std::optional` is not a viable alternative here.
+    atomic_bool is_initialized_{false};
 };
+#else
+// Subinterpreter support is enabled.
+// In this case, we should store the result per-interpreter instead of globally, because each
+// subinterpreter has its own separate state. The cached result may not shareable across
+// interpreters (e.g., imported modules and their members).
+//
+// The life span of the stored result is the entire interpreter lifetime. An additional
+// `finalize_fn` can be provided to clean up the stored result when the interpreter is destroyed.
+template <typename T>
+class gil_safe_call_once_and_store {
+public:
+    // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called.
+    template <typename Callable>
+    gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn,
+                                                             void (*finalize_fn)(T &) = nullptr) {
+        if (!is_last_storage_valid()) {
+            // Multiple threads may enter here, because the GIL is released in the next line and
+            // CPython API calls in the `fn()` call below may release and reacquire the GIL.
+            gil_scoped_release gil_rel; // Needed to establish lock ordering.
+            detail::with_internals([&](detail::internals &internals) {
+                const void *key = reinterpret_cast<const void *>(this);
+                auto &storage_map = internals.call_once_storage_map;
+                // There can be multiple threads going through here.
+                detail::call_once_storage<T> *value = nullptr;
+                {
+                    gil_scoped_acquire gil_acq;
+                    // Only one thread will enter here at a time.
+                    const auto it = storage_map.find(key);
+                    if (it != storage_map.end()) {
+                        value = static_cast<detail::call_once_storage<T> *>(it->second);
+                    } else {
+                        value = new detail::call_once_storage<T>{};
+                        storage_map.emplace(key, value);
+                    }
+                }
+                assert(value != nullptr);
+                std::call_once(value->once_flag, [&] {
+                    // Only one thread will ever enter here.
+                    gil_scoped_acquire gil_acq;
+                    // fn may release, but will reacquire, the GIL.
+                    ::new (value->storage) T(fn());
+                    value->finalize = finalize_fn;
+                    value->is_initialized = true;
+                    last_storage_ptr_ = reinterpret_cast<T *>(value->storage);
+                    is_initialized_by_atleast_one_interpreter_ = true;
+                });
+            });
+            // All threads will observe `is_initialized_by_atleast_one_interp_` as true here.
+        }
+        // Intentionally not returning `T &` to ensure the calling code is self-documenting.
+        return *this;
+    }
+
+    // This must only be called after `call_once_and_store_result()` was called.
+    T &get_stored() {
+        T *result = last_storage_ptr_;
+        if (!is_last_storage_valid()) {
+            detail::with_internals([&](detail::internals &internals) {
+                const void *key = reinterpret_cast<const void *>(this);
+                auto &storage_map = internals.call_once_storage_map;
+                auto *value = static_cast<detail::call_once_storage<T> *>(storage_map.at(key));
+                result = last_storage_ptr_ = reinterpret_cast<T *>(value->storage);
+            });
+        }
+        assert(result != nullptr);
+        return *result;
+    }
+
+    gil_safe_call_once_and_store() = default;
+    // The instance is a global static, so its destructor runs when the process
+    // is terminating. Therefore, do nothing here because the Python interpreter
+    // may have been finalized already.
+    PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default;
+
+private:
+    bool is_last_storage_valid() const {
+        return is_initialized_by_atleast_one_interpreter_
+               && detail::get_num_interpreters_seen() <= 1;
+    }
+
+    // No storage needed when subinterpreter support is enabled.
+    // The actual storage is stored in the per-interpreter state dict in
+    // `internals.call_once_storage_map`.
+
+    // Fast local cache to avoid repeated lookups when there are no multiple interpreters.
+    // This is only valid if there is a single interpreter. Otherwise, it is not used.
+    T *last_storage_ptr_ = nullptr;
+    // This flag is true if the value has been initialized by any interpreter (may not be the
+    // current one).
+    atomic_bool is_initialized_by_atleast_one_interpreter_{false};
+};
+#endif
 
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)