Uses ABx format with 16-bit signed offset

Optimizing cons-based list processing where nil marks end-of-list
Only handles nil? (not empty?) due to different semantics for arrays

Author: 0xhenrique

src/bytecode.rs

@@ -68,6 +68,8 @@ impl std::fmt::Debug for Op {
             Self::JUMP_IF_LE_IMM => write!(f, "JumpIfLeImm({}, {}, {})", self.a(), self.b() as i8, self.c() as i8),
             Self::JUMP_IF_GT_IMM => write!(f, "JumpIfGtImm({}, {}, {})", self.a(), self.b() as i8, self.c() as i8),
             Self::JUMP_IF_GE_IMM => write!(f, "JumpIfGeImm({}, {}, {})", self.a(), self.b() as i8, self.c() as i8),
+            Self::JUMP_IF_NIL => write!(f, "JumpIfNil({}, {})", self.a(), self.sbx()),
+            Self::JUMP_IF_NOT_NIL => write!(f, "JumpIfNotNil({}, {})", self.a(), self.sbx()),
             _ => write!(f, "Unknown(0x{:08x})", self.0),
         }
     }
@@ -124,6 +126,9 @@ impl Op {
     pub const JUMP_IF_LE_IMM: u8 = 45; // ABC: src, imm (i8), offset (i8) - jump if src <= imm
     pub const JUMP_IF_GT_IMM: u8 = 46; // ABC: src, imm (i8), offset (i8) - jump if src > imm
     pub const JUMP_IF_GE_IMM: u8 = 47; // ABC: src, imm (i8), offset (i8) - jump if src >= imm
+    // Specialized nil check opcodes (common in list processing)
+    pub const JUMP_IF_NIL: u8 = 48;    // A: src, sBx: offset - jump if src is nil
+    pub const JUMP_IF_NOT_NIL: u8 = 49; // A: src, sBx: offset - jump if src is NOT nil
 
     // ========== Constructors ==========
 
@@ -429,6 +434,17 @@ impl Op {
         Self::abc(Self::JUMP_IF_GE_IMM, src, imm as u8, offset as u8)
     }
 
+    // Specialized nil check (uses ABx format like JumpIfFalse for 16-bit offset)
+    #[inline(always)]
+    pub const fn jump_if_nil(src: Reg, offset: Offset) -> Self {
+        Self::asbx(Self::JUMP_IF_NIL, src, offset)
+    }
+
+    #[inline(always)]
+    pub const fn jump_if_not_nil(src: Reg, offset: Offset) -> Self {
+        Self::asbx(Self::JUMP_IF_NOT_NIL, src, offset)
+    }
+
     // ========== Jump patching helpers ==========
 
     /// Check if this is a jump instruction (for patching)
@@ -440,6 +456,7 @@ impl Op {
             || op == Self::JUMP_IF_GT || op == Self::JUMP_IF_GE
             || op == Self::JUMP_IF_LT_IMM || op == Self::JUMP_IF_LE_IMM
             || op == Self::JUMP_IF_GT_IMM || op == Self::JUMP_IF_GE_IMM
+            || op == Self::JUMP_IF_NIL || op == Self::JUMP_IF_NOT_NIL
     }
 
     /// Patch the offset of a jump instruction

src/compiler.rs

@@ -511,6 +511,11 @@ impl Compiler {
             return self.compile_if_branches(args, dest, tail_pos, jump_to_else);
         }
 
+        // Try to use specialized nil check opcodes
+        if let Some(jump_to_else) = self.try_compile_nil_check_jump(&args[0], dest)? {
+            return self.compile_if_branches(args, dest, tail_pos, jump_to_else);
+        }
+
         // Fallback: compile condition into dest and use JumpIfFalse
         self.compile_expr(&args[0], dest, false)?;
 
@@ -616,6 +621,38 @@ impl Compiler {
         Ok(Some(jump_pos))
     }
 
+    /// Try to compile a nil check condition as a specialized jump opcode.
+    /// Returns Some(jump_pos) if successful, None if condition is not a nil check.
+    ///
+    /// Handles pattern:
+    /// - `(nil? x)` → JumpIfNotNil (jump to else if x is NOT nil)
+    ///
+    /// Note: We don't optimize `(empty? x)` because it handles array-based lists
+    /// differently (empty array [] is not nil but is empty).
+    fn try_compile_nil_check_jump(&mut self, cond: &Value, dest: Reg) -> Result<Option<usize>, String> {
+        let items = match cond.as_list() {
+            Some(items) if items.len() == 2 => items,
+            _ => return Ok(None),
+        };
+
+        // Only optimize nil? - not empty? which has different semantics for array lists
+        if items[0].as_symbol() != Some("nil?") {
+            return Ok(None);
+        }
+
+        let arg = &items[1];
+
+        // Compile argument into dest
+        self.compile_expr(arg, dest, false)?;
+
+        // For `(if (nil? x) then else)`, we jump to else when condition is FALSE.
+        // Condition is TRUE when x IS nil.
+        // So we jump to else when x is NOT nil.
+        let jump_pos = self.emit(Op::jump_if_not_nil(dest, 0));
+
+        Ok(Some(jump_pos))
+    }
+
     fn compile_def(&mut self, args: &[Value], dest: Reg) -> Result<(), String> {
         if args.len() != 2 {
             return Err("def expects exactly 2 arguments".to_string());

src/vm.rs

@@ -889,6 +889,25 @@ impl VM {
                     }
                 }
 
+                // Specialized nil check opcodes
+                Op::JUMP_IF_NIL => {
+                    let src = instr.a();
+                    let offset = instr.sbx();
+                    let v = unsafe { self.registers.get_unchecked(base + src as usize) };
+                    if v.is_nil() {
+                        ip = (ip as isize + offset as isize) as usize;
+                    }
+                }
+
+                Op::JUMP_IF_NOT_NIL => {
+                    let src = instr.a();
+                    let offset = instr.sbx();
+                    let v = unsafe { self.registers.get_unchecked(base + src as usize) };
+                    if !v.is_nil() {
+                        ip = (ip as isize + offset as isize) as usize;
+                    }
+                }
+
                 _ => {
                     return Err(format!("Unknown opcode: {}", instr.opcode()));
                 }
@@ -1488,4 +1507,36 @@ mod tests {
         let result = vm_eval("(do (def sum (fn (n acc) (if (<= n 0) acc (sum (- n 1) (+ acc n))))) (sum 100 0))").unwrap();
         assert_eq!(result, Value::Int(5050));
     }
+
+    #[test]
+    fn test_nil_check_jump() {
+        // Test specialized nil check opcodes
+        // These are generated for (if (nil? x) ...) patterns
+
+        // Direct nil check
+        assert_eq!(vm_eval("(if (nil? nil) 1 2)").unwrap(), Value::Int(1));
+        assert_eq!(vm_eval("(if (nil? 42) 1 2)").unwrap(), Value::Int(2));
+
+        // Variable nil check
+        assert_eq!(vm_eval("(let (x nil) (if (nil? x) 1 2))").unwrap(), Value::Int(1));
+        assert_eq!(vm_eval("(let (x 42) (if (nil? x) 1 2))").unwrap(), Value::Int(2));
+
+        // Recursive list processing using nil check (with cons-based lists)
+        let result = vm_eval("(do
+            (def list-length (fn (lst)
+                (if (nil? lst)
+                    0
+                    (+ 1 (list-length (cdr lst))))))
+            (list-length (cons 1 (cons 2 (cons 3 (cons 4 (cons 5 nil)))))))").unwrap();
+        assert_eq!(result, Value::Int(5));
+
+        // List sum using nil? (with cons-based lists)
+        let result = vm_eval("(do
+            (def list-sum (fn (lst acc)
+                (if (nil? lst)
+                    acc
+                    (list-sum (cdr lst) (+ acc (car lst))))))
+            (list-sum (cons 1 (cons 2 (cons 3 (cons 4 (cons 5 nil))))) 0))").unwrap();
+        assert_eq!(result, Value::Int(15));
+    }
 }