Add spawn/join for thread creation - wip

Author: 0xhenrique

README.md

@@ -12,16 +12,6 @@ A Lisp interpreter with a register-based bytecode virtual machine.
 - **Immutable by default** - Values are immutable, inspired by FP
 - **Fast startup** - Minimal overhead, just feed the VM with code
 
-## Installation
-
-### Building from Source
-
-```bash
-~$ git clone https://github.com/0xhenrique/code-reg.git
-~$ cd reg
-~$ cargo build --release
-```
-
 ## How to use
 
 ### Running a Lisp File
@@ -109,7 +99,7 @@ Options:
 
 ### Recursion
 
-The VM supports kinda recursion with TCO. This is not battle tested:
+The VM kinda supports recursion with TCO. This is not battle tested:
 
 ```lisp
 ; Regular recursion
@@ -261,13 +251,14 @@ Macros enable compile-time code generation:
 
 The VM uses some common optimization techniques:
 
-- **Compile-time constant folding**: Expressions like `(+ 1 2 3)` compile directly to `6`
-- **Pure function inlining**: Pure functions with constant arguments are evaluated at compile time
+- **Constant folding**: Expressions like `(+ 1 2 3)` compile directly to `6`
+- **Function inlining**: Pure functions with constant arguments are evaluated at compile time
 - **Register-based bytecode**: Reduced memory traffic
 - **Specialized opcodes**: Direct arithmetic operations bypass function call overhead
 - **Tail call optimization**: Recursive tail calls reuse stack frames
 
-This is a rabbit hole by itself. I'm not expecting to beat something like Lua.
+Optimization is a rabbit hole by itself. I'm not expecting to beat something like Lua.
+For now, I will keep as is. Just don't try any memory intensive tests and it should work fine.
 
 ## Acknowledgments
 

lisp/test_spawn.lisp

@@ -0,0 +1,59 @@
+;; Test spawn and join - Phase 9 Concurrency
+
+(println "Testing spawn and join...")
+
+;; Test 1: Simple spawn/join - just return a value
+(println "\n=== Test 1: Simple constant ===")
+(def worker1 (fn () 42))
+(def handle1 (spawn worker1))
+(def result1 (join handle1))
+(println "Result:" result1)
+
+;; Test 2: Simple arithmetic
+(println "\n=== Test 2: Simple arithmetic ===")
+(def worker2 (fn () (+ (* 6 7) 10)))
+(def handle2 (spawn worker2))
+(def result2 (join handle2))
+(println "Result:" result2)
+
+;; Test 3: Working with lists (built-in functions only)
+(println "\n=== Test 3: List operations ===")
+(def worker3 (fn ()
+  (def nums (list 1 2 3 4 5))
+  (def first-item (car nums))
+  (def rest-items (cdr nums))
+  (+ first-item (car rest-items))))
+(def handle3 (spawn worker3))
+(def result3 (join handle3))
+(println "Result:" result3)
+
+;; Test 4: Multiple independent threads
+(println "\n=== Test 4: Three parallel computations ===")
+(def compute1 (fn () (+ 10 20 30)))
+(def compute2 (fn () (* 5 7)))
+(def compute3 (fn () (- 100 25)))
+
+(def t1 (spawn compute1))
+(def t2 (spawn compute2))
+(def t3 (spawn compute3))
+
+(println "Threads spawned, joining...")
+(def r1 (join t1))
+(def r2 (join t2))
+(def r3 (join t3))
+
+(println "Thread 1:" r1)
+(println "Thread 2:" r2)
+(println "Thread 3:" r3)
+(println "Sum:" (+ r1 r2 r3))
+
+;; Test 5: Thread returning a list
+(println "\n=== Test 5: Thread returning list ===")
+(def list-worker (fn ()
+  (list 1 2 3 4 5)))
+(def list-handle (spawn list-worker))
+(def list-result (join list-handle))
+(println "List from thread:" list-result)
+(println "Length:" (length list-result))
+
+(println "\n=== All spawn/join tests passed! ===")

src/value.rs

@@ -2,7 +2,8 @@ use std::cell::RefCell;
 use std::collections::HashMap;
 use std::fmt;
 use std::rc::Rc;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+use std::thread::JoinHandle;
 
 use crate::bytecode::Chunk;
 
@@ -260,6 +261,9 @@ pub enum HeapObject {
     NativeFunction(NativeFunction),
     /// Compiled function - keeps Rc<Chunk> for sharing in tail calls
     CompiledFunction(Rc<Chunk>),
+    /// Thread handle for spawned threads - wrapped in Mutex because JoinHandle can only be joined once
+    /// The Option allows us to take the handle when joining (join consumes the handle)
+    ThreadHandle(Arc<Mutex<Option<JoinHandle<Result<SharedValue, String>>>>>),
 }
 
 impl Clone for HeapObject {
@@ -272,6 +276,7 @@ impl Clone for HeapObject {
             HeapObject::Function(f) => HeapObject::Function(f.clone()),
             HeapObject::NativeFunction(f) => HeapObject::NativeFunction(f.clone()),
             HeapObject::CompiledFunction(c) => HeapObject::CompiledFunction(c.clone()),
+            HeapObject::ThreadHandle(h) => HeapObject::ThreadHandle(h.clone()),
         }
     }
 }
@@ -297,6 +302,8 @@ pub struct SharedConsCell {
 
 /// Thread-safe heap objects using Arc instead of Rc
 /// Mirrors HeapObject but is safe to send across threads
+/// Note: Functions are NOT included because closures capture environments with Rc
+/// and cannot be shared across threads. Use compiled functions or native functions instead.
 #[derive(Debug, Clone)]
 pub enum SharedHeapObject {
     /// String data
@@ -307,10 +314,6 @@ pub enum SharedHeapObject {
     List(Box<[SharedValue]>),
     /// Cons cell with shared values
     Cons(SharedConsCell),
-    /// Functions cannot be shared across threads (closures capture environment)
-    /// Use channels or other communication primitives instead
-    /// We keep this variant for completeness but it will error if attempted
-    Function(Box<Function>),
     /// Native functions are just function pointers, safe to share
     NativeFunction(NativeFunction),
     /// Compiled functions share their chunk via Arc
@@ -374,7 +377,6 @@ impl fmt::Display for SharedValue {
                 }
                 write!(f, ")")
             }
-            SharedHeapObject::Function(_) => write!(f, "<function>"),
             SharedHeapObject::NativeFunction(nf) => write!(f, "<native fn {}>", nf.name),
             SharedHeapObject::CompiledFunction(_) => write!(f, "<function>"),
         }
@@ -496,7 +498,7 @@ impl Value {
 
     /// Create a heap-allocated value from an Rc<HeapObject>
     /// Uses Rc::into_raw to store the pointer - refcount is NOT decremented
-    fn from_heap(heap: Rc<HeapObject>) -> Value {
+    pub fn from_heap(heap: Rc<HeapObject>) -> Value {
         let ptr = Rc::into_raw(heap) as u64;
         debug_assert!(ptr & TAG_MASK == 0, "Pointer uses more than 48 bits");
         Value(TAG_PTR | ptr)
@@ -597,7 +599,7 @@ impl Value {
 
     /// Get the heap object if this is a heap pointer (Rc or arena)
     #[inline]
-    fn as_heap(&self) -> Option<&HeapObject> {
+    pub fn as_heap(&self) -> Option<&HeapObject> {
         if self.is_ptr() {
             let ptr = (self.0 & PAYLOAD_MASK) as *const HeapObject;
             // Safety: we only create these pointers from Rc::into_raw
@@ -776,6 +778,7 @@ impl Value {
                 HeapObject::Function(_) => "function",
                 HeapObject::NativeFunction(_) => "native-function",
                 HeapObject::CompiledFunction(_) => "function",
+                HeapObject::ThreadHandle(_) => "thread-handle",
             }
         } else {
             "unknown"
@@ -959,6 +962,11 @@ impl Value {
                 Some(HeapObject::CompiledFunction(c)) => {
                     Value::CompiledFunction(c.clone())
                 }
+                Some(HeapObject::ThreadHandle(h)) => {
+                    // ThreadHandles are already Arc-based, just clone the Value
+                    let heap = Rc::new(HeapObject::ThreadHandle(h.clone()));
+                    Value::from_heap(heap)
+                }
                 None => Value::nil(),
             }
         } else if self.is_ptr() {
@@ -1073,6 +1081,9 @@ impl Value {
                     inner: Arc::new(SharedHeapObject::CompiledFunction(arc_chunk)),
                 })
             }
+            Some(HeapObject::ThreadHandle(_)) => {
+                Err("Thread handles cannot be shared across threads".to_string())
+            }
             None => Err("Cannot convert unknown value to SharedValue".to_string()),
         }
     }
@@ -1122,7 +1133,6 @@ impl Value {
                 let cdr = Value::from_shared(&cons.cdr);
                 Value::cons_rc(car, cdr)
             }
-            SharedHeapObject::Function(f) => Value::function((**f).clone()),
             SharedHeapObject::NativeFunction(nf) => {
                 Value::native_function(&nf.name, nf.func)
             }
@@ -1212,6 +1222,7 @@ impl fmt::Display for Value {
                 HeapObject::Function(_) => write!(f, "<function>"),
                 HeapObject::NativeFunction(nf) => write!(f, "<native fn {}>", nf.name),
                 HeapObject::CompiledFunction(_) => write!(f, "<function>"),
+                HeapObject::ThreadHandle(_) => write!(f, "<thread-handle>"),
             }
         } else {
             write!(f, "<unknown>")

src/vm.rs

@@ -1960,6 +1960,84 @@ pub fn standard_vm() -> VM {
             .map_err(|e| format!("write-file error: {}", e))
     }));
 
+    vm.define_global("spawn", native("spawn", |args| {
+        use std::sync::{Arc, Mutex};
+        use std::thread;
+
+        if args.len() != 1 {
+            return Err("spawn expects 1 argument (function)".to_string());
+        }
+
+        // Get the function to spawn
+        let func = &args[0];
+
+        // Only compiled functions can be spawned (not closures with captured environments)
+        let chunk = if let Some(chunk) = func.as_compiled_function() {
+            chunk.clone()
+        } else {
+            return Err("spawn expects a compiled function (not a closure)".to_string());
+        };
+
+        // Convert Rc<Chunk> to Arc<Chunk> for thread safety
+        let arc_chunk = Arc::new((*chunk).clone());
+
+        // Spawn the thread
+        let handle = thread::spawn(move || {
+            // Create a new VM for this thread
+            let mut thread_vm = standard_vm();
+
+            // Execute the function (it should be zero-argument)
+            match thread_vm.run((*arc_chunk).clone()) {
+                Ok(result) => {
+                    // Convert result to SharedValue
+                    result.make_shared()
+                }
+                Err(e) => Err(format!("Thread execution error: {}", e)),
+            }
+        });
+
+        // Wrap the JoinHandle in Arc<Mutex<Option<>>> so it can be joined once
+        let thread_handle = Arc::new(Mutex::new(Some(handle)));
+
+        // Create a HeapObject::ThreadHandle and wrap it in a Value
+        let heap = Rc::new(crate::value::HeapObject::ThreadHandle(thread_handle));
+        Ok(Value::from_heap(heap))
+    }));
+
+    vm.define_global("join", native("join", |args| {
+        use crate::value::HeapObject;
+
+        if args.len() != 1 {
+            return Err("join expects 1 argument (thread-handle)".to_string());
+        }
+
+        // Get the thread handle
+        let handle_obj = args[0].as_heap()
+            .ok_or_else(|| "join expects a thread-handle".to_string())?;
+
+        if let HeapObject::ThreadHandle(handle_mutex) = handle_obj {
+            // Take the handle from the mutex (can only join once)
+            let handle_opt = handle_mutex.lock()
+                .map_err(|_| "Failed to lock thread handle".to_string())?
+                .take();
+
+            let handle = handle_opt
+                .ok_or_else(|| "Thread already joined".to_string())?;
+
+            // Wait for the thread to complete
+            let result = handle.join()
+                .map_err(|_| "Thread panicked".to_string())?;
+
+            // Convert the result back from Result<SharedValue, String> to Value
+            match result {
+                Ok(shared_value) => Ok(Value::from_shared(&shared_value)),
+                Err(e) => Err(format!("Thread error: {}", e)),
+            }
+        } else {
+            Err("join expects a thread-handle".to_string())
+        }
+    }));
+
     vm
 }