PyO3 experiment

Cargo.toml

@@ -0,0 +1,10 @@
+[workspace]
+members = [
+    "c-questdb-client/questdb-rs-ffi",
+    "pystr-to-utf8",
+    "egress"
+]
+exclude = [
+    "c-questdb-client/system_test/tls_proxy"
+]
+resolver = "2"

RELEASING.rst

@@ -97,7 +97,7 @@ Inside the VM, open a terminal (or use the terminal Window in VSCode) and run th
     /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 \
         -m pip install -U pip
     /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 \
-        -m pip install -U setuptools wheel twine Cython cibuildwheel pandas numpy pyarrow
+        -m pip install -U setuptools setuptools-rust wheel twine Cython cibuildwheel pandas numpy pyarrow
 
 Smoke-testing the build
 -----------------------

ci/pip_install_deps.py

@@ -65,6 +65,7 @@ def main(args):
     ensure_timezone()
     pip_install('pip')
     pip_install('setuptools')
+    pip_install('setuptools-rust')
     try_pip_install('fastparquet>=2023.10.1')
 
     if args.pandas_version is not None and args.pandas_version != '':

dev_requirements.txt

@@ -3,10 +3,10 @@ Cython>=0.29.32
 wheel>=0.34.2
 cibuildwheel>=2.11.2
 Sphinx>=5.0.2
-sphinx-rtd-theme>=1.0.0
 twine>=4.0.1
 bump2version>=1.0.1
 pandas>=1.3.5
 numpy>=1.21.6
 pyarrow>=10.0.1
 fastparquet>=2023.10.1
+setuptools-rust>=1.9.0

egress/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "egress"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+pyo3 = { version = "0.21.1", features = ["extension-module"] }
+
+[lib]
+name = "questdb_egress"
+crate-type = ["cdylib"]

egress/build.rs

@@ -0,0 +1,9 @@
+use std::env;
+
+fn main() {
+    let target = env::var("TARGET").unwrap();
+    if target.contains("apple-darwin") {
+        println!("cargo:rustc-link-arg=-Wl,-undefined");
+        println!("cargo:rustc-link-arg=-Wl,dynamic_lookup");
+    }
+}

egress/src/lib.rs

@@ -0,0 +1,16 @@
+use pyo3::prelude::*;
+
+/// Formats the sum of two numbers as string.
+#[pyfunction]
+fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
+    Ok((a + b).to_string())
+}
+
+/// A Python module implemented in Rust. The name of this function must match
+/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
+/// import the module.
+#[pymodule]
+fn egress(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
+    Ok(())
+}

proj.py

@@ -8,7 +8,6 @@
 import subprocess
 import shlex
 import pathlib
-import glob
 import platform
 
 PROJ_ROOT = pathlib.Path(__file__).parent
@@ -233,11 +232,13 @@ def sdist():
 
 @command
 def clean():
+    _rmtree(PROJ_ROOT / 'target')
     _rmtree(PROJ_ROOT / 'build')
     _rmtree(PROJ_ROOT / 'dist')
     _rmtree(PROJ_ROOT / 'c-questdb-client' / 'questdb-rs-ffi' / 'target')
     _rmtree(PROJ_ROOT / 'c-questdb-client' / 'build')
     _rmtree(PROJ_ROOT / 'pystr-to-utf8' / 'target')
+    _rmtree(PROJ_ROOT / 'egress' / 'target')
     _rmtree(PROJ_ROOT / 'src' / 'questdb.egg-info')
     _rmtree(PROJ_ROOT / 'venv')
     _rmtree(PROJ_ROOT / 'wheelhouse')

pyproject.toml

@@ -42,10 +42,10 @@ Slack = "http://slack.questdb.io"
 
 [build-system]
 requires = [
-    # Setuptools 18.0 and above properly handles Cython extensions.
-    "setuptools>=45.2.0",
-    "wheel>=0.34.2",
-    "cython>=0.29.24"]
+    "setuptools>=69.5.1",
+    "wheel>=0.43.0",
+    "cython>=3.0.10",
+    "setuptools-rust>=1.9.0"]
 
 
 [tool.cibuildwheel]

setup.py

@@ -6,6 +6,7 @@
 import shutil
 import platform
 
+from setuptools_rust import RustExtension
 from setuptools import setup, find_packages
 from setuptools.extension import Extension
 from setuptools.command.build_ext import build_ext
@@ -39,18 +40,12 @@ def ingress_extension():
     extra_link_args = []
     extra_objects = []
 
-    questdb_rs_ffi_dir = PROJ_ROOT / 'c-questdb-client' / 'questdb-rs-ffi'
-    pystr_to_utf8_dir = PROJ_ROOT / 'pystr-to-utf8'
-    questdb_client_lib_dir = None
-    pystr_to_utf8_lib_dir = None
+    lib_dir = None
     if PLATFORM == 'win32' and MODE == '32bit':
-        questdb_client_lib_dir = \
-            questdb_rs_ffi_dir / 'target' / WIN_32BIT_CARGO_TARGET / 'release'
-        pystr_to_utf8_lib_dir = \
-            pystr_to_utf8_dir / 'target' / WIN_32BIT_CARGO_TARGET / 'release'
+        lib_dir = \
+            PROJ_ROOT / 'target' / WIN_32BIT_CARGO_TARGET / 'release'
     else:
-        questdb_client_lib_dir = questdb_rs_ffi_dir / 'target' / 'release'
-        pystr_to_utf8_lib_dir = pystr_to_utf8_dir / 'target' / 'release'
+        lib_dir = PROJ_ROOT / 'target' / 'release'
         if INSTRUMENT_FUZZING:
             extra_compile_args.append('-fsanitize=fuzzer-no-link')
             extra_link_args.append('-fsanitize=fuzzer-no-link')
@@ -75,8 +70,8 @@ def ingress_extension():
     extra_objects = [
         str(loc / f'{lib_prefix}{name}{lib_suffix}')
         for loc, name in (
-            (questdb_client_lib_dir, 'questdb_client'),
-            (pystr_to_utf8_lib_dir, 'pystr_to_utf8'))]
+            (lib_dir, 'questdb_client'),
+            (lib_dir, 'pystr_to_utf8'))]
 
     return Extension(
         "questdb.ingress",
@@ -137,14 +132,13 @@ def cargo_build():
             env['CXX'] = ORIG_CXX
         else:
             del env['CXX']
-    subprocess.check_call(
-        cargo_args + ['--features', 'confstr-ffi'],
-        cwd=str(PROJ_ROOT / 'c-questdb-client' / 'questdb-rs-ffi'),
-        env=env)
 
     subprocess.check_call(
-        cargo_args,
-        cwd=str(PROJ_ROOT / 'pystr-to-utf8'),
+        cargo_args + [
+            '--package', 'pystr-to-utf8',
+            '--package', 'questdb-rs-ffi',
+            '--features', 'questdb-rs-ffi/confstr-ffi'],
+        cwd=str(PROJ_ROOT),
         env=env)
 
 
@@ -169,9 +163,11 @@ def readme():
     platforms=['any'],
     python_requires='>=3.8',
     install_requires=[],
-    ext_modules = cythonize([ingress_extension()], annotate=True),
+    ext_modules=cythonize([ingress_extension()], annotate=True),
+    rust_extensions=[
+        RustExtension("questdb.egress", path="egress/Cargo.toml")],
     cmdclass={'build_ext': questdb_build_ext},
-    zip_safe = False,
+    zip_safe=False,
     package_dir={'': 'src'},
     test_suite="tests",
     packages=find_packages('src', exclude=['test']))

test/test.py

@@ -17,6 +17,7 @@
 from mock_server import Server, HttpServer
 
 import questdb.ingress as qi
+import questdb.egress as qe
 
 if os.environ.get('TEST_QUESTDB_INTEGRATION') == '1':
     from system_test import TestWithDatabase
@@ -1152,6 +1153,13 @@ class TestSenderEnv(TestBases.TestSender):
     builder = Builder.ENV
 
 
+class TestEgress(unittest.TestCase):
+    def test_sum_as_string(self):
+        exp = '1234'
+        act = qe.sum_as_string(1030, 204)
+        self.assertEqual(act, exp)
+
+
 if __name__ == '__main__':
     if os.environ.get('TEST_QUESTDB_PROFILE') == '1':
         import cProfile