Merge 'Initial pass on loadable rust extensions' from Preston Thorpe

This PR adds the start of an implementation of an extension library for
`limbo` so users can write extensions in rust, that can be loaded at
runtime with the `.load` cli command.
The existing "ExtensionFunc"  `uuid`, has been replaced with the first
complete limbo extension in `extensions/uuid`
![image](https://github.com/user-
attachments/assets/63aa06cc-9390-4277-ba09-3a9be5524535)
![image](https://github.com/user-
attachments/assets/75899748-5e26-406a-84ee-26063383afeb)
There is still considerable work to do on this, as this only implements
scalar functions, but this PR is already plenty big enough.
Design + implementation comments or suggestions would be appreciated
:+1:
I tried out using `abi_stable`, so that trait objects and other goodies
could be used across FFI bounds, but to be honest I didn't find it too
much better than this. I personally haven't done a whole lot with FFI,
or anything at all linking dynamically in Rust, so if there is something
I seem to be missing here, please let me know.
I added some tests, similar to how shell-tests are setup.
If anyone can test this on other platforms, that would be helpful as
well as I am limited to x86_64 linux here

Closes #658

Author: penberg

Cargo.lock

@@ -11,7 +11,7 @@ members = [
     "sqlite3",
     "core",
     "simulator",
-    "test", "macros",
+    "test", "macros", "limbo_extension", "extensions/uuid",
 ]
 exclude = ["perf/latency/limbo"]
 

Cargo.toml

@@ -62,10 +62,15 @@ limbo-wasm:
 	cargo build --package limbo-wasm --target wasm32-wasi
 .PHONY: limbo-wasm
 
-test: limbo test-compat test-sqlite3 test-shell
+test: limbo test-compat test-sqlite3 test-shell test-extensions
 .PHONY: test
 
-test-shell: limbo
+test-extensions: limbo
+	cargo build --package limbo_uuid
+	./testing/extensions.py
+.PHONY: test-extensions
+
+test-shell: limbo 
 	SQLITE_EXEC=$(SQLITE_EXEC) ./testing/shelltests.py
 .PHONY: test-shell
 

Makefile

@@ -129,6 +129,8 @@ pub enum Command {
     Tables,
     /// Import data from FILE into TABLE
     Import,
+    /// Loads an extension library
+    LoadExtension,
 }
 
 impl Command {
@@ -141,7 +143,12 @@ impl Command {
             | Self::ShowInfo
             | Self::Tables
             | Self::SetOutput => 0,
-            Self::Open | Self::OutputMode | Self::Cwd | Self::Echo | Self::NullValue => 1,
+            Self::Open
+            | Self::OutputMode
+            | Self::Cwd
+            | Self::Echo
+            | Self::NullValue
+            | Self::LoadExtension => 1,
             Self::Import => 2,
         } + 1) // argv0
     }
@@ -160,6 +167,7 @@ impl Command {
             Self::NullValue => ".nullvalue <string>",
             Self::Echo => ".echo on|off",
             Self::Tables => ".tables",
+            Self::LoadExtension => ".load",
             Self::Import => &IMPORT_HELP,
         }
     }
@@ -182,6 +190,7 @@ impl FromStr for Command {
             ".nullvalue" => Ok(Self::NullValue),
             ".echo" => Ok(Self::Echo),
             ".import" => Ok(Self::Import),
+            ".load" => Ok(Self::LoadExtension),
             _ => Err("Unknown command".to_string()),
         }
     }
@@ -314,6 +323,11 @@ impl Limbo {
         };
     }
 
+    #[cfg(not(target_family = "wasm"))]
+    fn handle_load_extension(&mut self, path: &str) -> Result<(), String> {
+        self.conn.load_extension(path).map_err(|e| e.to_string())
+    }
+
     fn display_in_memory(&mut self) -> std::io::Result<()> {
         if self.opts.db_file == ":memory:" {
             self.writeln("Connected to a transient in-memory database.")?;
@@ -537,6 +551,13 @@ impl Limbo {
                         let _ = self.writeln(e.to_string());
                     };
                 }
+                Command::LoadExtension =>
+                {
+                    #[cfg(not(target_family = "wasm"))]
+                    if let Err(e) = self.handle_load_extension(args[1]) {
+                        let _ = self.writeln(&e);
+                    }
+                }
             }
         } else {
             let _ = self.write_fmt(format_args!(

cli/app.rs

@@ -35,6 +35,7 @@ rustix = "0.38.34"
 mimalloc = { version = "*", default-features = false }
 
 [dependencies]
+limbo_extension = { path = "../limbo_extension" }
 cfg_block = "0.1.1"
 fallible-iterator = "0.3.0"
 hex = "0.4.3"
@@ -58,6 +59,7 @@ bumpalo = { version = "3.16.0", features = ["collections", "boxed"] }
 limbo_macros = { path = "../macros" }
 uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true }
 miette = "7.4.0"
+libloading = "0.8.6"
 
 [target.'cfg(not(target_family = "windows"))'.dev-dependencies]
 pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] }

core/Cargo.toml

@@ -39,6 +39,8 @@ pub enum LimboError {
     InvalidModifier(String),
     #[error("Runtime error: {0}")]
     Constraint(String),
+    #[error("Extension error: {0}")]
+    ExtensionError(String),
 }
 
 #[macro_export]

core/error.rs

@@ -1,37 +1,44 @@
-#[cfg(feature = "uuid")]
-mod uuid;
-#[cfg(feature = "uuid")]
-pub use uuid::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc};
+use crate::{function::ExternalFunc, Database};
+use limbo_extension::{ExtensionApi, ResultCode, ScalarFunction, RESULT_ERROR, RESULT_OK};
+pub use limbo_extension::{Value as ExtValue, ValueType as ExtValueType};
+use std::{
+    ffi::{c_char, c_void, CStr},
+    rc::Rc,
+};
 
-#[derive(Debug, Clone, PartialEq)]
-pub enum ExtFunc {
-    #[cfg(feature = "uuid")]
-    Uuid(UuidFunc),
+extern "C" fn register_scalar_function(
+    ctx: *mut c_void,
+    name: *const c_char,
+    func: ScalarFunction,
+) -> ResultCode {
+    let c_str = unsafe { CStr::from_ptr(name) };
+    let name_str = match c_str.to_str() {
+        Ok(s) => s.to_string(),
+        Err(_) => return RESULT_ERROR,
+    };
+    if ctx.is_null() {
+        return RESULT_ERROR;
+    }
+    let db = unsafe { &*(ctx as *const Database) };
+    db.register_scalar_function_impl(name_str, func)
 }
 
-#[allow(unreachable_patterns)] // TODO: remove when more extension funcs added
-impl std::fmt::Display for ExtFunc {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            #[cfg(feature = "uuid")]
-            Self::Uuid(uuidfn) => write!(f, "{}", uuidfn),
-            _ => write!(f, "unknown"),
-        }
+impl Database {
+    fn register_scalar_function_impl(&self, name: String, func: ScalarFunction) -> ResultCode {
+        self.syms.borrow_mut().functions.insert(
+            name.to_string(),
+            Rc::new(ExternalFunc {
+                name: name.to_string(),
+                func,
+            }),
+        );
+        RESULT_OK
     }
-}
 
-#[allow(unreachable_patterns)]
-impl ExtFunc {
-    pub fn resolve_function(name: &str, num_args: usize) -> Option<ExtFunc> {
-        match name {
-            #[cfg(feature = "uuid")]
-            name => UuidFunc::resolve_function(name, num_args),
-            _ => None,
+    pub fn build_limbo_extension(&self) -> ExtensionApi {
+        ExtensionApi {
+            ctx: self as *const _ as *mut c_void,
+            register_scalar_function,
         }
     }
 }
-
-pub fn init(db: &mut crate::Database) {
-    #[cfg(feature = "uuid")]
-    uuid::init(db);
-}