Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v2.12] Backport: Enable type-safe interoperability between different independent Python/C++ bindings systems. #5368

Merged
merged 6 commits into from
Sep 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ endif()
set(PYBIND11_HEADERS
include/pybind11/detail/class.h
include/pybind11/detail/common.h
include/pybind11/detail/cpp_conduit.h
include/pybind11/detail/descr.h
include/pybind11/detail/init.h
include/pybind11/detail/internals.h
Expand Down
77 changes: 77 additions & 0 deletions include/pybind11/detail/cpp_conduit.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Copyright (c) 2024 The pybind Community.

#pragma once

#include <pybind11/pytypes.h>

#include "common.h"
#include "internals.h"

#include <typeinfo>

PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
PYBIND11_NAMESPACE_BEGIN(detail)

// Forward declaration needed here: Refactoring opportunity.
extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *);

inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) {
#if defined(PYPY_VERSION)
auto &internals = get_internals();
return bool(internals.registered_types_py.find(type_obj)
!= internals.registered_types_py.end());
#else
return bool(type_obj->tp_new == pybind11_object_new);
#endif
}

inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) {
PyObject *descr = _PyType_Lookup(type_obj, attr_name);
return bool((descr != nullptr) && PyInstanceMethod_Check(descr));
}

inline object try_get_cpp_conduit_method(PyObject *obj) {
if (PyType_Check(obj)) {
return object();
}
PyTypeObject *type_obj = Py_TYPE(obj);
str attr_name("_pybind11_conduit_v1_");
bool assumed_to_be_callable = false;
if (type_is_managed_by_our_internals(type_obj)) {
if (!is_instance_method_of_type(type_obj, attr_name.ptr())) {
return object();
}
assumed_to_be_callable = true;
}
PyObject *method = PyObject_GetAttr(obj, attr_name.ptr());
if (method == nullptr) {
PyErr_Clear();
return object();
}
if (!assumed_to_be_callable && PyCallable_Check(method) == 0) {
Py_DECREF(method);
return object();
}
return reinterpret_steal<object>(method);
}

inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src,
const std::type_info *cpp_type_info) {
object method = try_get_cpp_conduit_method(src.ptr());
if (method) {
capsule cpp_type_info_capsule(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
typeid(std::type_info).name());
object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID),
cpp_type_info_capsule,
bytes("raw_pointer_ephemeral"));
if (isinstance<capsule>(cpp_conduit)) {
return reinterpret_borrow<capsule>(cpp_conduit).get_pointer();
}
}
return nullptr;
}

#define PYBIND11_HAS_CPP_CONDUIT 1

PYBIND11_NAMESPACE_END(detail)
PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
10 changes: 6 additions & 4 deletions include/pybind11/detail/internals.h
Original file line number Diff line number Diff line change
Expand Up @@ -316,15 +316,17 @@ struct type_info {
# endif
#endif

#define PYBIND11_PLATFORM_ABI_ID \
PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \
PYBIND11_BUILD_TYPE

#define PYBIND11_INTERNALS_ID \
"__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \
PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \
PYBIND11_BUILD_TYPE "__"
PYBIND11_PLATFORM_ABI_ID "__"

#define PYBIND11_MODULE_LOCAL_ID \
"__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \
PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \
PYBIND11_BUILD_TYPE "__"
PYBIND11_PLATFORM_ABI_ID "__"

/// Each module locally stores a pointer to the `internals` data. The data
/// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`.
Expand Down
40 changes: 40 additions & 0 deletions include/pybind11/detail/type_caster_base.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@

#include "../pytypes.h"
#include "common.h"
#include "cpp_conduit.h"
#include "descr.h"
#include "internals.h"
#include "typeid.h"

#include <cstdint>
#include <cstring>
#include <iterator>
#include <new>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <typeindex>
Expand Down Expand Up @@ -674,6 +677,13 @@ class type_caster_generic {
}
return false;
}
bool try_cpp_conduit(handle src) {
value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype);
if (value != nullptr) {
return true;
}
return false;
}
void check_holder_compat() {}

PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) {
Expand Down Expand Up @@ -805,6 +815,10 @@ class type_caster_generic {
return true;
}

if (convert && cpptype && this_.try_cpp_conduit(src)) {
return true;
}

return false;
}

Expand Down Expand Up @@ -832,6 +846,32 @@ class type_caster_generic {
void *value = nullptr;
};

inline object cpp_conduit_method(handle self,
const bytes &pybind11_platform_abi_id,
const capsule &cpp_type_info_capsule,
const bytes &pointer_kind) {
#ifdef PYBIND11_HAS_STRING_VIEW
using cpp_str = std::string_view;
#else
using cpp_str = std::string;
#endif
if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) {
return none();
}
if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) {
return none();
}
if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") {
throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\"");
}
const auto *cpp_type_info = cpp_type_info_capsule.get_pointer<const std::type_info>();
type_caster_generic caster(*cpp_type_info);
if (!caster.load(self, false)) {
return none();
}
return capsule(caster.value, cpp_type_info->name());
}

/**
* Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster
* needs to provide `operator T*()` and `operator T&()` operators.
Expand Down
7 changes: 5 additions & 2 deletions include/pybind11/pybind11.h
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,8 @@ class cpp_function : public function {
int index = 0;
/* Create a nice pydoc rec including all signatures and
docstrings of the functions in the overload chain */
if (chain && options::show_function_signatures()) {
if (chain && options::show_function_signatures()
&& std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
// First a generic signature
signatures += rec->name;
signatures += "(*args, **kwargs)\n";
Expand All @@ -619,7 +620,8 @@ class cpp_function : public function {
// Then specific overload signatures
bool first_user_def = true;
for (auto *it = chain_start; it != nullptr; it = it->next) {
if (options::show_function_signatures()) {
if (options::show_function_signatures()
&& std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
if (index > 0) {
signatures += '\n';
}
Expand Down Expand Up @@ -1617,6 +1619,7 @@ class class_ : public detail::generic_type {
instances[std::type_index(typeid(type_alias))]
= instances[std::type_index(typeid(type))];
}
def("_pybind11_conduit_v1_", cpp_conduit_method);
}

template <typename Base, detail::enable_if_t<is_base<Base>::value, int> = 0>
Expand Down
3 changes: 3 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ set(PYBIND11_TEST_FILES
test_const_name
test_constants_and_functions
test_copy_move
test_cpp_conduit
test_custom_type_casters
test_custom_type_setup
test_docstring_options
Expand Down Expand Up @@ -220,6 +221,8 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_
# And add additional targets for other tests.
tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set")
tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils")
tests_extra_targets("test_cpp_conduit.py"
"exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler")

set(PYBIND11_EIGEN_REPO
"https://gitlab.com/libeigen/eigen.git"
Expand Down
103 changes: 103 additions & 0 deletions tests/exo_planet_c_api.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright (c) 2024 The pybind Community.

// THIS MUST STAY AT THE TOP!
#include <pybind11/pybind11.h> // EXCLUSIVELY for PYBIND11_PLATFORM_ABI_ID
// Potential future direction to maximize reusability:
// (e.g. for use from SWIG, Cython, PyCLIF, nanobind):
// #include <pybind11/compat/platform_abi_id.h>
// This would only depend on:
// 1. A C++ compiler, WITHOUT requiring -fexceptions.
// 2. Python.h

#include "test_cpp_conduit_traveler_types.h"

#include <Python.h>
#include <typeinfo>

namespace {

void *get_cpp_conduit_void_ptr(PyObject *py_obj, const std::type_info *cpp_type_info) {
PyObject *cpp_type_info_capsule
= PyCapsule_New(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
typeid(std::type_info).name(),
nullptr);
if (cpp_type_info_capsule == nullptr) {
return nullptr;
}
PyObject *cpp_conduit = PyObject_CallMethod(py_obj,
"_pybind11_conduit_v1_",
"yOy",
PYBIND11_PLATFORM_ABI_ID,
cpp_type_info_capsule,
"raw_pointer_ephemeral");
Py_DECREF(cpp_type_info_capsule);
if (cpp_conduit == nullptr) {
return nullptr;
}
void *void_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name());
Py_DECREF(cpp_conduit);
if (PyErr_Occurred()) {
return nullptr;
}
return void_ptr;
}

template <typename T>
T *get_cpp_conduit_type_ptr(PyObject *py_obj) {
void *void_ptr = get_cpp_conduit_void_ptr(py_obj, &typeid(T));
if (void_ptr == nullptr) {
return nullptr;
}
return static_cast<T *>(void_ptr);
}

extern "C" PyObject *wrapGetLuggage(PyObject * /*self*/, PyObject *traveler) {
const auto *cpp_traveler
= get_cpp_conduit_type_ptr<pybind11_tests::test_cpp_conduit::Traveler>(traveler);
if (cpp_traveler == nullptr) {
return nullptr;
}
return PyUnicode_FromString(cpp_traveler->luggage.c_str());
}

extern "C" PyObject *wrapGetPoints(PyObject * /*self*/, PyObject *premium_traveler) {
const auto *cpp_premium_traveler
= get_cpp_conduit_type_ptr<pybind11_tests::test_cpp_conduit::PremiumTraveler>(
premium_traveler);
if (cpp_premium_traveler == nullptr) {
return nullptr;
}
return PyLong_FromLong(static_cast<long>(cpp_premium_traveler->points));
}

PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr},
{"GetPoints", wrapGetPoints, METH_O, nullptr},
{nullptr, nullptr, 0, nullptr}};

struct PyModuleDef ThisModuleDef = {
PyModuleDef_HEAD_INIT, // m_base
"exo_planet_c_api", // m_name
nullptr, // m_doc
-1, // m_size
ThisMethodDef, // m_methods
nullptr, // m_slots
nullptr, // m_traverse
nullptr, // m_clear
nullptr // m_free
};

} // namespace

#if defined(WIN32) || defined(_WIN32)
# define EXO_PLANET_C_API_EXPORT __declspec(dllexport)
#else
# define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default")))
#endif

extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() {
PyObject *m = PyModule_Create(&ThisModuleDef);
if (m == nullptr) {
return nullptr;
}
return m;
}
19 changes: 19 additions & 0 deletions tests/exo_planet_pybind11.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// Copyright (c) 2024 The pybind Community.

#if defined(PYBIND11_INTERNALS_VERSION)
# undef PYBIND11_INTERNALS_VERSION
#endif
#define PYBIND11_INTERNALS_VERSION 900000001

#include "test_cpp_conduit_traveler_bindings.h"

namespace pybind11_tests {
namespace test_cpp_conduit {

PYBIND11_MODULE(exo_planet_pybind11, m) {
wrap_traveler(m);
m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
}

} // namespace test_cpp_conduit
} // namespace pybind11_tests
1 change: 1 addition & 0 deletions tests/extra_python_package/test_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
detail_headers = {
"include/pybind11/detail/class.h",
"include/pybind11/detail/common.h",
"include/pybind11/detail/cpp_conduit.h",
"include/pybind11/detail/descr.h",
"include/pybind11/detail/init.h",
"include/pybind11/detail/internals.h",
Expand Down
13 changes: 13 additions & 0 deletions tests/home_planet_very_lonely_traveler.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Copyright (c) 2024 The pybind Community.

#include "test_cpp_conduit_traveler_bindings.h"

namespace pybind11_tests {
namespace test_cpp_conduit {

PYBIND11_MODULE(home_planet_very_lonely_traveler, m) {
m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
}

} // namespace test_cpp_conduit
} // namespace pybind11_tests
22 changes: 22 additions & 0 deletions tests/test_cpp_conduit.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright (c) 2024 The pybind Community.

#include "pybind11_tests.h"
#include "test_cpp_conduit_traveler_bindings.h"

#include <typeinfo>

namespace pybind11_tests {
namespace test_cpp_conduit {

TEST_SUBMODULE(cpp_conduit, m) {
m.attr("PYBIND11_PLATFORM_ABI_ID") = py::bytes(PYBIND11_PLATFORM_ABI_ID);
m.attr("cpp_type_info_capsule_Traveler")
= py::capsule(&typeid(Traveler), typeid(std::type_info).name());
m.attr("cpp_type_info_capsule_int") = py::capsule(&typeid(int), typeid(std::type_info).name());

wrap_traveler(m);
wrap_lonely_traveler(m);
}

} // namespace test_cpp_conduit
} // namespace pybind11_tests
Loading
Loading