Add per-interpreter storage for gil_safe_call_once_and_store

include/pybind11/detail/class.h

@@ -226,14 +226,12 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) {
                 local_internals.registered_types_cpp.erase(tinfo->cpptype);
             } else {
                 internals.registered_types_cpp.erase(tindex);
-#if PYBIND11_INTERNALS_VERSION >= 12
                 internals.registered_types_cpp_fast.erase(tinfo->cpptype);
                 for (const std::type_info *alias : tinfo->alias_chain) {
                     auto num_erased = internals.registered_types_cpp_fast.erase(alias);
                     (void) num_erased;
                     assert(num_erased > 0);
                 }
-#endif
             }
             internals.registered_types_py.erase(tinfo->type);
 

include/pybind11/detail/internals.h

@@ -39,11 +39,11 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    define PYBIND11_INTERNALS_VERSION 11
+#    define PYBIND11_INTERNALS_VERSION 12
 #endif
 
-#if PYBIND11_INTERNALS_VERSION < 11
-#    error "PYBIND11_INTERNALS_VERSION 11 is the minimum for all platforms for pybind11v3."
+#if PYBIND11_INTERNALS_VERSION < 12
+#    error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11v3."
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -234,6 +234,38 @@ inline uint64_t round_up_to_next_pow2(uint64_t x) {
 
 class loader_life_support;
 
+struct call_once_storage_base {
+    call_once_storage_base() = default;
+    virtual ~call_once_storage_base() = default;
+    call_once_storage_base(const call_once_storage_base &) = delete;
+    call_once_storage_base(call_once_storage_base &&) = delete;
+    call_once_storage_base &operator=(const call_once_storage_base &) = delete;
+    call_once_storage_base &operator=(call_once_storage_base &&) = delete;
+};
+
+template <typename T>
+struct call_once_storage : call_once_storage_base {
+    alignas(T) char storage[sizeof(T)] = {};
+    std::once_flag once_flag;
+    void (*finalize)(T &) = nullptr;
+    std::atomic_bool is_initialized{false};
+
+    call_once_storage() = default;
+    ~call_once_storage() override {
+        if (is_initialized) {
+            if (finalize != nullptr) {
+                finalize(*reinterpret_cast<T *>(storage));
+            } else {
+                reinterpret_cast<T *>(storage)->~T();
+            }
+        }
+    };
+    call_once_storage(const call_once_storage &) = delete;
+    call_once_storage(call_once_storage &&) = delete;
+    call_once_storage &operator=(const call_once_storage &) = delete;
+    call_once_storage &operator=(call_once_storage &&) = delete;
+};
+
 /// Internal data structure used to track registered instances and types.
 /// Whenever binary incompatible changes are made to this structure,
 /// `PYBIND11_INTERNALS_VERSION` must be incremented.
@@ -242,14 +274,12 @@ struct internals {
     pymutex mutex;
     pymutex exception_translator_mutex;
 #endif
-#if PYBIND11_INTERNALS_VERSION >= 12
     // non-normative but fast "hint" for registered_types_cpp. Meant
     // to be used as the first level of a two-level lookup: successful
     // lookups are correct, but unsuccessful lookups need to try
     // registered_types_cpp and then backfill this map if they find
     // anything.
     fast_type_map<type_info *> registered_types_cpp_fast;
-#endif
 
     // std::type_index -> pybind11's type information
     type_map<type_info *> registered_types_cpp;
@@ -275,14 +305,13 @@ struct internals {
     PyObject *instance_base = nullptr;
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     thread_specific_storage<PyThreadState> tstate;
-#if PYBIND11_INTERNALS_VERSION <= 11
-    thread_specific_storage<loader_life_support> loader_life_support_tls; // OBSOLETE (PR #5830)
-#endif
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PyInterpreterState *istate = nullptr;
 
     type_map<PyObject *> native_enum_type_map;
 
+    std::unordered_map<const void *, call_once_storage_base *> call_once_storage_map;
+
     internals()
         : static_property_type(make_static_property_type()),
           default_metaclass(make_default_metaclass()) {
@@ -308,7 +337,12 @@ struct internals {
     internals(internals &&other) = delete;
     internals &operator=(const internals &other) = delete;
     internals &operator=(internals &&other) = delete;
-    ~internals() = default;
+    ~internals() {
+        for (const auto &entry : call_once_storage_map) {
+            delete entry.second;
+        }
+        call_once_storage_map.clear();
+    }
 };
 
 // the internals struct (above) is shared between all the modules. local_internals are only
@@ -358,7 +392,6 @@ struct type_info {
     void *(*module_local_load)(PyObject *, const type_info *) = nullptr;
     holder_enum_t holder_enum_v = holder_enum_t::undefined;
 
-#if PYBIND11_INTERNALS_VERSION >= 12
     // When a type appears in multiple DSOs,
     // internals::registered_types_cpp_fast will have multiple distinct
     // keys (the std::type_info from each DSO) mapped to the same
@@ -369,7 +402,6 @@ struct type_info {
     // nb_alias_chain` added in
     // https://github.com/wjakob/nanobind/commit/b515b1f7f2f4ecc0357818e6201c94a9f4cbfdc2
     std::forward_list<const std::type_info *> alias_chain;
-#endif
 
     /* A simple type never occurs as a (direct or indirect) parent
      * of a class that makes use of multiple inheritance.
@@ -564,6 +596,15 @@ class internals_pp_manager {
     /// acquire the GIL. Will never return nullptr.
     std::unique_ptr<InternalsType> *get_pp() {
 #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
+        // FIXME: We cannot use `get_num_interpreters_seen() > 1` here to create a fast path for
+        // the multi-interpreter case. The singleton may be initialized by a subinterpreter not the
+        // main interpreter.
+        //
+        // For multi-interpreter support, the subinterpreters can be initialized concurrently, and
+        // the first time this function may not be called in the main interpreter.
+        // For example, a clean main interpreter that does not import any pybind11 module and then
+        // spawns multiple subinterpreters using `InterpreterPoolExecutor` that each imports a
+        // pybind11 module concurrently.
         if (get_num_interpreters_seen() > 1) {
             // Whenever the interpreter changes on the current thread we need to invalidate the
             // internals_pp so that it can be pulled from the interpreter's state dict.  That is
@@ -580,15 +621,29 @@ class internals_pp_manager {
             return internals_p_tls();
         }
 #endif
-        if (!internals_singleton_pp_) {
-            gil_scoped_acquire_simple gil;
-            internals_singleton_pp_ = get_or_create_pp_in_state_dict();
+        return get_pp_for_main_interpreter();
+    }
+
+    /// Get the pointer-to-pointer for the main interpreter, allocating it if it does not already
+    /// exist.  May acquire the GIL. Will never return nullptr.
+    std::unique_ptr<InternalsType> *get_pp_for_main_interpreter() {
+        if (!seen_main_interpreter_) {
+            // The first call to this function **MUST** be from the main interpreter.
+            // Here we **ASSUME** that the current thread is running in the main interpreter.
+            // The caller is responsible for ensuring this.
+            std::call_once(seen_main_interpreter_flag_, [&] {
+                gil_scoped_acquire_simple gil;
+                internals_singleton_pp_ = get_or_create_pp_in_state_dict();
+                seen_main_interpreter_ = true;
+            });
         }
+        // This is shared between all threads and all interpreters.
         return internals_singleton_pp_;
     }
 
     /// Drop all the references we're currently holding.
     void unref() {
+        // See comment in get_pp() above.
 #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
         if (get_num_interpreters_seen() > 1) {
             last_istate_tls() = nullptr;
@@ -600,6 +655,7 @@ class internals_pp_manager {
     }
 
     void destroy() {
+        // See comment in get_pp() above.
 #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
         if (get_num_interpreters_seen() > 1) {
             auto *tstate = get_thread_state_unchecked();
@@ -660,7 +716,10 @@ class internals_pp_manager {
 
     char const *holder_id_ = nullptr;
     on_fetch_function *on_fetch_ = nullptr;
+    // Pointer to the singleton internals for the main interpreter
     std::unique_ptr<InternalsType> *internals_singleton_pp_;
+    std::once_flag seen_main_interpreter_flag_;
+    std::atomic_bool seen_main_interpreter_{false};
 };
 
 // If We loaded the internals through `state_dict`, our `error_already_set`

include/pybind11/detail/type_caster_base.h

@@ -227,32 +227,26 @@ inline detail::type_info *get_global_type_info_lock_held(const std::type_info &t
     // next time.
     detail::type_info *type_info = nullptr;
     auto &internals = get_internals();
-#if PYBIND11_INTERNALS_VERSION >= 12
     auto &fast_types = internals.registered_types_cpp_fast;
-#endif
     auto &types = internals.registered_types_cpp;
-#if PYBIND11_INTERNALS_VERSION >= 12
     auto fast_it = fast_types.find(&tp);
     if (fast_it != fast_types.end()) {
-#    ifndef NDEBUG
+#ifndef NDEBUG
         auto types_it = types.find(std::type_index(tp));
         assert(types_it != types.end());
         assert(types_it->second == fast_it->second);
-#    endif
+#endif
         return fast_it->second;
     }
-#endif // PYBIND11_INTERNALS_VERSION >= 12
 
     auto it = types.find(std::type_index(tp));
     if (it != types.end()) {
-#if PYBIND11_INTERNALS_VERSION >= 12
         // We found the type in the slow map but not the fast one, so
         // some other DSO added it (otherwise it would be in the fast
         // map under &tp) and therefore we must be an alias. Record
         // that.
         it->second->alias_chain.push_front(&tp);
         fast_types.emplace(&tp, it->second);
-#endif
         type_info = it->second;
     }
     return type_info;