From e8aec1a83d3a6a9f806b2c12fa7b7b35a48fd787 Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 09:38:35 +0530 Subject: [PATCH 01/15] pass correct btree flags for without rowid tables --- core/translate/schema.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/core/translate/schema.rs b/core/translate/schema.rs index 6356f049ac..ac7b3c7d83 100644 --- a/core/translate/schema.rs +++ b/core/translate/schema.rs @@ -187,10 +187,25 @@ pub fn translate_create_table( // TODO: SetCookie let table_root_reg = program.alloc_register(); + + let btree_flags = if let ast::CreateTableBody::ColumnsAndConstraints { options, .. } = &body { + if options.contains(ast::TableOptions::WITHOUT_ROWID) { + tracing::debug!( + "CREATE TABLE: `{}` is a WITHOUT ROWID table, using index b-tree flags.", + normalized_tbl_name + ); + CreateBTreeFlags::new_index() + } else { + CreateBTreeFlags::new_table() + } + } else { + CreateBTreeFlags::new_table() + }; + program.emit_insn(Insn::CreateBtree { db: 0, root: table_root_reg, - flags: CreateBTreeFlags::new_table(), + flags: btree_flags, }); // Create an automatic index B-tree if needed From 32867f47b9c9d78bed65368a853351c4d155f6f4 Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 09:39:13 +0530 Subject: [PATCH 02/15] dont create autoindexes for withoutrowid tables --- core/translate/schema.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/core/translate/schema.rs b/core/translate/schema.rs index ac7b3c7d83..c329d39ccb 100644 --- a/core/translate/schema.rs +++ b/core/translate/schema.rs @@ -429,6 +429,17 @@ fn collect_autoindexes( program: &mut ProgramBuilder, tbl_name: &str, ) -> Result>> { + let is_without_rowid = if let ast::CreateTableBody::ColumnsAndConstraints { options, .. } = body + { + options.contains(ast::TableOptions::WITHOUT_ROWID) + } else { + false + }; + + if is_without_rowid { + tracing::debug!("Without rowid table, no need for autoindex'{}'", tbl_name); + } + let table = create_table(tbl_name, body, 0)?; let mut regs: Vec = Vec::new(); @@ -441,7 +452,7 @@ fn collect_autoindexes( }; let needs_index = if us.is_primary_key { - !(col.primary_key() && col.is_rowid_alias()) + !is_without_rowid && !(col.primary_key() && col.is_rowid_alias()) } else { // UNIQUE single needs an index true @@ -452,7 +463,11 @@ fn collect_autoindexes( } } - for _us in table.unique_sets.iter().filter(|us| us.columns.len() > 1) { + for _us in table + .unique_sets + .iter() + .filter(|us| us.columns.len() > 1 && (!us.is_primary_key || !is_without_rowid)) + { regs.push(program.alloc_register()); } if regs.is_empty() { From 6e8946eae89a4ef6c752ef0897d15abb944e6f1b Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 10:15:56 +0530 Subject: [PATCH 03/15] add a stub for without rowid insert --- core/translate/insert.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 880e47399e..b24bd90547 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -74,9 +74,9 @@ fn validate(table_name: &str, resolver: &Resolver, table: &Table) -> Result<()> "INSERT to table with indexes is disabled. Omit the `--experimental-indexes=false` flag to enable this feature." ); } - if table.btree().is_some_and(|t| !t.has_rowid) { - crate::bail_parse_error!("INSERT into WITHOUT ROWID table is not supported"); - } + // if table.btree().is_some_and(|t| !t.has_rowid) { + // crate::bail_parse_error!("INSERT into WITHOUT ROWID table is not supported"); + // } Ok(()) } @@ -179,6 +179,23 @@ impl<'a> InsertEmitCtx<'a> { } } +#[allow(clippy::too_many_arguments)] +fn translate_without_rowid_insert( + resolver: &Resolver, + on_conflict: Option, + table: &Arc, + columns: Vec, + mut body: InsertBody, + mut returning: Vec, + mut program: ProgramBuilder, + connection: &Arc, +) -> Result { + crate::bail_parse_error!( + "INSERT into WITHOUT ROWID table '{}' is not yet implemented.", + table.name + ); +} + #[allow(clippy::too_many_arguments)] pub fn translate_insert( resolver: &Resolver, @@ -221,6 +238,18 @@ pub fn translate_insert( crate::bail_parse_error!("no such table: {}", table_name); }; + if !btree_table.has_rowid { + return translate_without_rowid_insert( + resolver, + on_conflict, + &btree_table, + columns, + body, + returning, + program, + connection, + ); + } let BoundInsertResult { mut values, mut upsert_actions, From 891a7e883fca288d32458e066fb36a8d1350aadb Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 13:46:47 +0530 Subject: [PATCH 04/15] add insert for without rowid --- core/incremental/operator.rs | 1 + core/schema.rs | 27 +++ core/storage/btree.rs | 20 ++ core/translate/compound_select.rs | 1 + core/translate/index.rs | 1 + core/translate/insert.rs | 360 ++++++++++++++++++++++++++++-- core/translate/main_loop.rs | 2 + core/translate/optimizer/mod.rs | 1 + core/translate/order_by.rs | 1 + core/translate/subquery.rs | 1 + core/vdbe/execute.rs | 11 + 11 files changed, 409 insertions(+), 17 deletions(-) diff --git a/core/incremental/operator.rs b/core/incremental/operator.rs index 5d577a5523..0914e41849 100644 --- a/core/incremental/operator.rs +++ b/core/incremental/operator.rs @@ -71,6 +71,7 @@ pub fn create_dbsp_state_index(root_page: i64) -> Index { has_rowid: true, where_clause: None, index_method: None, + is_primary_key: false, } } diff --git a/core/schema.rs b/core/schema.rs index 4990b4b3c8..191adf9d37 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -751,7 +751,29 @@ impl Schema { } } + let has_rowid = table.has_rowid; + let table_name_for_index = table.name.clone(); + let root_page_for_index = table.root_page; + self.add_btree_table(Arc::new(table))?; + + if !has_rowid { + tracing::debug!( + "WITHOUT ROWID table '{}' found. Synthesizing primary key index.", + &table_name_for_index + ); + let table_ref = self.get_btree_table(&table_name_for_index).unwrap(); + let pk_index = Index::automatic_from_primary_key( + &table_ref, + + ( + format!("sqlite_autoindex_{}_1", &table_name_for_index), + root_page_for_index, + ), + table_ref.primary_key_columns.len(), + )?; + self.add_index(Arc::new(pk_index))?; + } } } "index" => { @@ -2393,6 +2415,7 @@ pub struct Index { pub has_rowid: bool, pub where_clause: Option>, pub index_method: Option>, + pub is_primary_key: bool, } #[allow(dead_code)] @@ -2459,6 +2482,7 @@ impl Index { ephemeral: false, has_rowid: table.has_rowid, where_clause: None, + is_primary_key: false, index_method: Some(descriptor), }) } else { @@ -2472,6 +2496,7 @@ impl Index { has_rowid: table.has_rowid, where_clause, index_method: None, + is_primary_key: false, }) } } @@ -2526,6 +2551,7 @@ impl Index { has_rowid: table.has_rowid, where_clause: None, index_method: None, + is_primary_key: true, }) } @@ -2564,6 +2590,7 @@ impl Index { has_rowid: table.has_rowid, where_clause: None, index_method: None, + is_primary_key: false, }) } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 84bfc4992b..37bf27a1a1 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5044,6 +5044,26 @@ impl CursorTrait for BTreeCursor { #[instrument(skip_all, level = Level::DEBUG)] fn insert(&mut self, key: &BTreeKey) -> Result> { + // remove this after getting sanity, before merging + + match key { + BTreeKey::TableRowId((rowid, record)) => { + tracing::debug!( + "BTreeCursor::insert: root_page={}, key_type=TableRowId, rowid={}, record_size={}", + self.root_page, + rowid, + record.map_or(0, |r| r.get_payload().len()) + ); + } + BTreeKey::IndexKey(record) => { + tracing::debug!( + "BTreeCursor::insert: root_page={}, key_type=IndexKey, record_size={}", + self.root_page, + record.get_payload().len() + ); + } + } + tracing::debug!(valid_state = ?self.valid_state, cursor_state = ?self.state, is_write_in_progress = self.is_write_in_progress()); return_if_io!(self.insert_into_page(key)); if key.maybe_rowid().is_some() { diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 96f1afda2f..19f9e4918e 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -448,6 +448,7 @@ fn create_dedupe_index( has_rowid: false, where_clause: None, index_method: None, + is_primary_key: false, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(dedupe_index.clone())); program.emit_insn(Insn::OpenEphemeral { diff --git a/core/translate/index.rs b/core/translate/index.rs index 44e8efbb84..83aa56eb27 100644 --- a/core/translate/index.rs +++ b/core/translate/index.rs @@ -149,6 +149,7 @@ pub fn translate_create_index( // before translating, and it cannot reference a table alias where_clause: where_clause.clone(), index_method: index_method.clone(), + is_primary_key: false, }); if !idx.validate_where_expr(table) { diff --git a/core/translate/insert.rs b/core/translate/insert.rs index b24bd90547..9d2d4fcc44 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -179,23 +179,6 @@ impl<'a> InsertEmitCtx<'a> { } } -#[allow(clippy::too_many_arguments)] -fn translate_without_rowid_insert( - resolver: &Resolver, - on_conflict: Option, - table: &Arc, - columns: Vec, - mut body: InsertBody, - mut returning: Vec, - mut program: ProgramBuilder, - connection: &Arc, -) -> Result { - crate::bail_parse_error!( - "INSERT into WITHOUT ROWID table '{}' is not yet implemented.", - table.name - ); -} - #[allow(clippy::too_many_arguments)] pub fn translate_insert( resolver: &Resolver, @@ -2645,3 +2628,346 @@ pub fn emit_parent_side_fk_decrement_on_insert( } Ok(()) } + +fn build_without_rowid_insertion<'a>( + program: &mut ProgramBuilder, + table: &'a BTreeTable, + columns: &'a [ast::Name], + num_values: usize, +) -> Result> { + let table_columns = &table.columns; + // for without rowid, there is no separate rowid, so we allocate a dummy register. + // the real key is the composite primary key. + let dummy_rowid_register = program.alloc_register(); + + let mut column_mappings = table_columns + .iter() + .map(|c| ColMapping { + column: c, + value_index: None, + register: program.alloc_register(), + }) + .collect::>(); + + if columns.is_empty() { + if num_values != table_columns.len() { + crate::bail_parse_error!( + "table {} has {} columns but {} values were supplied", + &table.name, + table_columns.len(), + num_values + ); + } + for (i, _col) in table_columns.iter().enumerate() { + column_mappings[i].value_index = Some(i); + } + } else { + if num_values != columns.len() { + crate::bail_parse_error!( + "table {} has {} columns but {} values were supplied", + &table.name, + columns.len(), + num_values + ); + } + for (value_index, column_name) in columns.iter().enumerate() { + let column_name_str = normalize_ident(column_name.as_str()); + if let Some((idx_in_table, _)) = table.get_column(&column_name_str) { + column_mappings[idx_in_table].value_index = Some(value_index); + } else { + crate::bail_parse_error!( + "table {} has no column named {}", + &table.name, + column_name_str + ); + } + } + } + + Ok(Insertion { + key: InsertionKey::Autogenerated { + register: dummy_rowid_register, + }, + col_mappings: column_mappings, + record_reg: program.alloc_register(), + }) +} + +#[allow(clippy::too_many_arguments)] +fn translate_without_rowid_insert( + resolver: &Resolver, + on_conflict: Option, + table: &Arc, + columns: Vec, + mut body: InsertBody, + mut returning: Vec, + mut program: ProgramBuilder, + connection: &Arc, +) -> Result { + tracing::debug!( + "Beginning translation for WITHOUT ROWID INSERT into table '{}'", + table.name + ); + + let BoundInsertResult { + values, + upsert_actions, + inserting_multiple_rows, + } = bind_insert( + &mut program, + resolver, + &Table::BTree(table.clone()), + &mut body, + connection, + on_conflict.unwrap_or(ResolveType::Abort), + )?; + + if inserting_multiple_rows { + crate::bail_parse_error!( + "Multi-row INSERT into WITHOUT ROWID tables is not yet implemented." + ); + } + // TODO: Add support for UPSERT clauses with WITHOUT ROWID tables. + if !upsert_actions.is_empty() { + crate::bail_parse_error!( + "ON CONFLICT clause is not yet supported for WITHOUT ROWID tables." + ); + } + + let cdc_table = prepare_cdc_if_necessary(&mut program, resolver.schema, table.name.as_str())?; + let (result_columns, _) = process_returning_clause( + &mut returning, + &Table::BTree(table.clone()), + table.name.as_str(), + &mut program, + connection, + )?; + + let mut ctx = InsertEmitCtx::new( + &mut program, + resolver, + table, + on_conflict, + cdc_table, + values.len(), + None, + )?; + + let pk_index = resolver + .schema + .get_indices(&table.name) + .find(|idx| { + if idx.columns.len() != table.primary_key_columns.len() { + return false; + } + idx.columns + .iter() + .zip(table.primary_key_columns.iter()) + .all(|(ic, (pkc_name, _))| ic.name == *pkc_name) + }) + .ok_or_else(|| { + crate::error::LimboError::InternalError(format!( + "WITHOUT ROWID table {} has no primary key index", + table.name + )) + })?; + + tracing::debug!( + "Found primary key index '{}' with root page {}", + pk_index.name, + pk_index.root_page + ); + + let pk_cursor_id = program.alloc_cursor_index(None, pk_index)?; + ctx.cursor_id = pk_cursor_id; + + program.emit_insn(Insn::OpenWrite { + cursor_id: pk_cursor_id, + root_page: RegisterOrLiteral::Literal(pk_index.root_page), + db: 0, + }); + tracing::debug!( + "Emitted OP_OpenWrite for PK index on cursor {}", + pk_cursor_id + ); + + for (name, root_page, cursor_id) in &ctx.idx_cursors { + if *root_page != pk_index.root_page { + program.emit_insn(Insn::OpenWrite { + cursor_id: *cursor_id, + root_page: RegisterOrLiteral::Literal(*root_page), + db: 0, + }); + tracing::debug!( + "Emitted OP_OpenWrite for secondary index '{}' on cursor {}", + name, + cursor_id + ); + } + } + + let insertion = build_without_rowid_insertion(&mut program, table, &columns, values.len())?; + translate_rows_single(&mut program, &values, &insertion, resolver)?; + tracing::debug!( + "Translated values into registers starting at {}", + insertion.first_col_register() + ); + + emit_notnulls(&mut program, &ctx, &insertion); + + let pk_columns = &pk_index.columns; + let num_pk_cols = pk_columns.len(); + let pk_registers_start = program.alloc_registers(num_pk_cols); + tracing::debug!( + "Allocated registers {}-{} for PK uniqueness check", + pk_registers_start, + pk_registers_start + num_pk_cols - 1 + ); + + for (i, pk_col) in pk_columns.iter().enumerate() { + let mapping = insertion.get_col_mapping_by_name(&pk_col.name).unwrap(); + program.emit_insn(Insn::Copy { + src_reg: mapping.register, + dst_reg: pk_registers_start + i, + extra_amount: 0, + }); + } + + let ok_to_insert_label = program.allocate_label(); + program.emit_insn(Insn::NoConflict { + cursor_id: pk_cursor_id, + target_pc: ok_to_insert_label, + record_reg: pk_registers_start, + num_regs: num_pk_cols, + }); + tracing::debug!( + "Emitted OP_NoConflict to check for key uniqueness on cursor {}", + pk_cursor_id + ); + + program.emit_insn(Insn::Halt { + err_code: SQLITE_CONSTRAINT_UNIQUE, + description: format_unique_violation_desc(&table.name, pk_index), + }); + + program.preassign_label_to_next_insn(ok_to_insert_label); + tracing::debug!("Uniqueness check passed. Continuing with insertion."); + + let full_record_reg = program.alloc_register(); + let affinity_str = insertion + .col_mappings + .iter() + .map(|col_mapping| col_mapping.column.affinity().aff_mask()) + .collect::(); + + program.emit_insn(Insn::MakeRecord { + start_reg: insertion.first_col_register(), + count: insertion.col_mappings.len(), + dest_reg: full_record_reg, + index_name: Some(pk_index.name.clone()), + affinity_str: Some(affinity_str), + }); + tracing::debug!( + "Emitted OP_MakeRecord for full row into register {}", + full_record_reg + ); + + program.emit_insn(Insn::IdxInsert { + cursor_id: pk_cursor_id, + record_reg: full_record_reg, + unpacked_start: Some(insertion.first_col_register()), + unpacked_count: Some(insertion.col_mappings.len() as u16), + flags: IdxInsertFlags::new().nchange(true), + }); + tracing::debug!( + "Emitted OP_IdxInsert to write record from reg {} into cursor {}", + full_record_reg, + pk_cursor_id + ); + + tracing::debug!("Beginning update of secondary indices."); + for index in resolver.schema.get_indices(table.name.as_str()) { + let is_pk = index.columns.len() == pk_index.columns.len() + && index + .columns + .iter() + .zip(pk_index.columns.iter()) + .all(|(a, b)| a.name == b.name); + if is_pk { + continue; + } + + let idx_cursor_id = ctx + .idx_cursors + .iter() + .find(|(name, _, _)| *name == &index.name) + .map(|(_, _, c_id)| *c_id) + .expect("no cursor found for secondary index"); + + tracing::debug!("Processing secondary index '{}'", index.name); + + let secondary_key_cols = &index.columns; + let num_key_cols = secondary_key_cols.len() + pk_columns.len(); + let key_start_reg = program.alloc_registers(num_key_cols); + + let mut current_reg = key_start_reg; + + for idx_col in secondary_key_cols { + let mapping = insertion.get_col_mapping_by_name(&idx_col.name).unwrap(); + program.emit_insn(Insn::Copy { + src_reg: mapping.register, + dst_reg: current_reg, + extra_amount: 0, + }); + current_reg += 1; + } + + for pk_col in pk_columns { + let mapping = insertion.get_col_mapping_by_name(&pk_col.name).unwrap(); + program.emit_insn(Insn::Copy { + src_reg: mapping.register, + dst_reg: current_reg, + extra_amount: 0, + }); + current_reg += 1; + } + + let record_reg = program.alloc_register(); + program.emit_insn(Insn::MakeRecord { + start_reg: key_start_reg, + count: num_key_cols, + dest_reg: record_reg, + index_name: Some(index.name.clone()), + affinity_str: None, + }); + tracing::debug!( + "Emitted OP_MakeRecord for secondary index '{}' key into register {}", + index.name, + record_reg + ); + + program.emit_insn(Insn::IdxInsert { + cursor_id: idx_cursor_id, + record_reg, + unpacked_start: Some(key_start_reg), + unpacked_count: Some(num_key_cols as u16), + flags: IdxInsertFlags::new().nchange(true), + }); + tracing::debug!("Emitted OP_IdxInsert for secondary index '{}'", index.name); + } + + // TODO? + if !result_columns.is_empty() { + crate::bail_parse_error!("RETURNING clause is not yet supported for WITHOUT ROWID tables."); + } + + program.emit_insn(Insn::Goto { + target_pc: ctx.row_done_label, + }); + + emit_epilogue(&mut program, &ctx, inserting_multiple_rows); + + program.set_needs_stmt_subtransactions(true); + tracing::debug!("Finished translation for WITHOUT ROWID INSERT."); + Ok(program) +} diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 14424ea7fb..0f19564f12 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -108,6 +108,7 @@ pub fn init_distinct(program: &mut ProgramBuilder, plan: &SelectPlan) -> Result< has_rowid: false, where_clause: None, index_method: None, + is_primary_key: false, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); let ctx = DistinctCtx { @@ -186,6 +187,7 @@ pub fn init_loop( unique: false, where_clause: None, index_method: None, + is_primary_key: false, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); if group_by.is_none() { diff --git a/core/translate/optimizer/mod.rs b/core/translate/optimizer/mod.rs index cc706b898d..dc1e8e9d5c 100644 --- a/core/translate/optimizer/mod.rs +++ b/core/translate/optimizer/mod.rs @@ -1309,6 +1309,7 @@ fn ephemeral_index_build( .btree() .is_some_and(|btree| btree.has_rowid), index_method: None, + is_primary_key: false, }; ephemeral_index diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index c450c80df4..018cd77e89 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -110,6 +110,7 @@ pub fn init_order_by( has_rowid: false, where_clause: None, index_method: None, + is_primary_key: false, }); program.alloc_cursor_id(CursorType::BTreeIndex(index)) } else { diff --git a/core/translate/subquery.rs b/core/translate/subquery.rs index 51999b9c4e..06ca1ce9cd 100644 --- a/core/translate/subquery.rs +++ b/core/translate/subquery.rs @@ -392,6 +392,7 @@ fn get_subquery_parser<'a>( unique: false, where_clause: None, index_method: None, + is_primary_key: false, }); let cursor_id = diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index bba8b9cc98..c09e919f86 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -6690,6 +6690,17 @@ pub fn op_idx_insert( *insn ); + // remove this after getting sanity, before merging + let (_, cursor_type) = program.cursor_ref.get(cursor_id).unwrap(); + if let CursorType::BTreeIndex(index_meta) = cursor_type { + tracing::debug!( + "Executing OP_IdxInsert: cursor={}, index='{}', record_reg={}", + cursor_id, + index_meta.name, + record_reg + ); + } + if let Some(Cursor::IndexMethod(cursor)) = &mut state.cursors[cursor_id] { let Some(start) = unpacked_start else { return Err(LimboError::InternalError( From 53c9eb96ac240c0c684dada1213d84d4dc44e4a6 Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 14:07:24 +0530 Subject: [PATCH 05/15] add is_primary_key flag to tests --- core/schema.rs | 1 - core/storage/btree.rs | 2 ++ core/translate/optimizer/join.rs | 8 ++++++++ withoutrow | Bin 0 -> 8192 bytes withoutrow-shm | Bin 0 -> 32768 bytes 5 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 withoutrow create mode 100644 withoutrow-shm diff --git a/core/schema.rs b/core/schema.rs index 191adf9d37..29d7908f39 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -765,7 +765,6 @@ impl Schema { let table_ref = self.get_btree_table(&table_name_for_index).unwrap(); let pk_index = Index::automatic_from_primary_key( &table_ref, - ( format!("sqlite_autoindex_{}_1", &table_name_for_index), root_page_for_index, diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 37bf27a1a1..26fbb9ef23 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -8578,6 +8578,7 @@ mod tests { ephemeral: false, has_rowid: false, index_method: None, + is_primary_key: false, }; let num_columns = index_def.columns.len(); let mut cursor = @@ -8738,6 +8739,7 @@ mod tests { ephemeral: false, has_rowid: false, index_method: None, + is_primary_key: false, }; let mut cursor = BTreeCursor::new_index(pager.clone(), index_root_page, &index_def, 1); diff --git a/core/translate/optimizer/join.rs b/core/translate/optimizer/join.rs index 70a7b8ac32..8a445003d8 100644 --- a/core/translate/optimizer/join.rs +++ b/core/translate/optimizer/join.rs @@ -688,6 +688,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, + is_primary_key: false, }); available_indexes.insert("test_table".to_string(), VecDeque::from([index])); @@ -762,6 +763,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, + is_primary_key: false, }); available_indexes.insert("table1".to_string(), VecDeque::from([index1])); @@ -884,6 +886,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, + is_primary_key: false, }); available_indexes.insert(table_name.to_string(), VecDeque::from([index])); }); @@ -903,6 +906,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, + is_primary_key: false, }); let order_id_idx = Arc::new(Index { name: "order_items_order_id_idx".to_string(), @@ -920,6 +924,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, + is_primary_key: false, }); available_indexes @@ -1360,6 +1365,7 @@ mod tests { ephemeral: false, has_rowid: true, index_method: None, + is_primary_key: false, }); let mut available_indexes = HashMap::new(); @@ -1459,6 +1465,7 @@ mod tests { ephemeral: false, has_rowid: true, index_method: None, + is_primary_key: false, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); @@ -1576,6 +1583,7 @@ mod tests { has_rowid: true, unique: false, index_method: None, + is_primary_key: false, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); diff --git a/withoutrow b/withoutrow new file mode 100644 index 0000000000000000000000000000000000000000..e3371248e36bf895ea46eb773ef623cb5683ed29 GIT binary patch literal 8192 zcmeIuF$=;l5Cz~%6bC_SC)XQw5s9;4)qurHwN`4!3RMI}kPJ@#H~*D`P*c#=!BO7h z-f%}C@U51?!Q{L>v<; z00bZa0SG_<0uX=z1Rwwb2>c>25Ve-;ioMCU$J{8R Date: Sun, 2 Nov 2025 14:15:09 +0530 Subject: [PATCH 06/15] remove databases --- withoutrow | Bin 8192 -> 0 bytes withoutrow-shm | Bin 32768 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 withoutrow delete mode 100644 withoutrow-shm diff --git a/withoutrow b/withoutrow deleted file mode 100644 index e3371248e36bf895ea46eb773ef623cb5683ed29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeIuF$=;l5Cz~%6bC_SC)XQw5s9;4)qurHwN`4!3RMI}kPJ@#H~*D`P*c#=!BO7h z-f%}C@U51?!Q{L>v<; z00bZa0SG_<0uX=z1Rwwb2>c>25Ve-;ioMCU$J{8R Date: Sun, 2 Nov 2025 14:22:30 +0530 Subject: [PATCH 07/15] clippy and fmt --- core/storage/btree.rs | 4 ++-- core/translate/insert.rs | 2 +- core/translate/optimizer/join.rs | 16 ++++++++-------- core/translate/schema.rs | 3 ++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 26fbb9ef23..74e861877f 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -8578,7 +8578,7 @@ mod tests { ephemeral: false, has_rowid: false, index_method: None, - is_primary_key: false, + is_primary_key: false, }; let num_columns = index_def.columns.len(); let mut cursor = @@ -8739,7 +8739,7 @@ mod tests { ephemeral: false, has_rowid: false, index_method: None, - is_primary_key: false, + is_primary_key: false, }; let mut cursor = BTreeCursor::new_index(pager.clone(), index_root_page, &index_def, 1); diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 9d2d4fcc44..10b7b96b3a 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -44,7 +44,7 @@ use super::plan::QueryDestination; use super::select::translate_select; /// Validate anything with this insert statement that should throw an early parse error -fn validate(table_name: &str, resolver: &Resolver, table: &Table) -> Result<()> { +fn validate(table_name: &str, resolver: &Resolver, _table: &Table) -> Result<()> { // Check if this is a system table that should be protected from direct writes if crate::schema::is_system_table(table_name) { crate::bail_parse_error!("table {} may not be modified", table_name); diff --git a/core/translate/optimizer/join.rs b/core/translate/optimizer/join.rs index 8a445003d8..72eb84309e 100644 --- a/core/translate/optimizer/join.rs +++ b/core/translate/optimizer/join.rs @@ -688,7 +688,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes.insert("test_table".to_string(), VecDeque::from([index])); @@ -763,7 +763,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes.insert("table1".to_string(), VecDeque::from([index1])); @@ -886,7 +886,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes.insert(table_name.to_string(), VecDeque::from([index])); }); @@ -906,7 +906,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); let order_id_idx = Arc::new(Index { name: "order_items_order_id_idx".to_string(), @@ -924,7 +924,7 @@ mod tests { root_page: 1, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes @@ -1365,7 +1365,7 @@ mod tests { ephemeral: false, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); let mut available_indexes = HashMap::new(); @@ -1465,7 +1465,7 @@ mod tests { ephemeral: false, has_rowid: true, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); @@ -1583,7 +1583,7 @@ mod tests { has_rowid: true, unique: false, index_method: None, - is_primary_key: false, + is_primary_key: false, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); diff --git a/core/translate/schema.rs b/core/translate/schema.rs index c329d39ccb..a297b74fe4 100644 --- a/core/translate/schema.rs +++ b/core/translate/schema.rs @@ -452,7 +452,8 @@ fn collect_autoindexes( }; let needs_index = if us.is_primary_key { - !is_without_rowid && !(col.primary_key() && col.is_rowid_alias()) + + !(is_without_rowid || (col.primary_key() && col.is_rowid_alias())) } else { // UNIQUE single needs an index true From fc885a6347f41b546807cd8b547a9e98248ec0bb Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 17:39:01 +0530 Subject: [PATCH 08/15] use is_primary_key flag instead of comapring --- core/schema.rs | 33 +++++++++++++++++++++++++++------ core/translate/insert.rs | 10 +--------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/core/schema.rs b/core/schema.rs index 29d7908f39..6ff23445e7 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -2520,31 +2520,52 @@ impl Index { assert!(has_primary_key_index); let (index_name, root_page) = auto_index; - let mut primary_keys = Vec::with_capacity(column_count); + let mut index_columns = Vec::new(); + let mut key_column_positions = std::collections::HashSet::new(); + for (col_name, order) in table.primary_key_columns.iter() { - let Some((pos_in_table, _)) = table.get_column(col_name) else { + let Some((pos_in_table, column)) = table.get_column(col_name) else { return Err(crate::LimboError::ParseError(format!( "Column {} not found in table {}", col_name, table.name ))); }; - let (_, column) = table.get_column(col_name).unwrap(); - primary_keys.push(IndexColumn { + index_columns.push(IndexColumn { name: normalize_ident(col_name), order: *order, pos_in_table, collation: column.collation_opt(), default: column.default.clone(), }); + key_column_positions.insert(pos_in_table); } - assert!(primary_keys.len() == column_count); + assert!( + index_columns.len() == column_count, + "Mismatch in primary key column count" + ); + + if !table.has_rowid { + for (pos_in_table, column) in table.columns.iter().enumerate() { + // TODO: when we support generated cols look at this + + if !key_column_positions.contains(&pos_in_table) { + index_columns.push(IndexColumn { + name: normalize_ident(column.name.as_ref().unwrap()), + order: SortOrder::Asc, + pos_in_table, + collation: None, + default: column.default.clone(), + }); + } + } + } Ok(Index { name: normalize_ident(index_name.as_str()), table_name: table.name.clone(), root_page, - columns: primary_keys, + columns: index_columns, unique: true, ephemeral: false, has_rowid: table.has_rowid, diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 10b7b96b3a..c783242d7a 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2756,15 +2756,7 @@ fn translate_without_rowid_insert( let pk_index = resolver .schema .get_indices(&table.name) - .find(|idx| { - if idx.columns.len() != table.primary_key_columns.len() { - return false; - } - idx.columns - .iter() - .zip(table.primary_key_columns.iter()) - .all(|(ic, (pkc_name, _))| ic.name == *pkc_name) - }) + .find(|idx| idx.is_primary_key) .ok_or_else(|| { crate::error::LimboError::InternalError(format!( "WITHOUT ROWID table {} has no primary key index", From 94a634be2be4c825c0b9fe9319d580101a89506c Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 17:39:47 +0530 Subject: [PATCH 09/15] integrity_check: ignore rootpages for withoutrowid tables --- core/translate/integrity_check.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/translate/integrity_check.rs b/core/translate/integrity_check.rs index c4ee30289f..f39656e33d 100644 --- a/core/translate/integrity_check.rs +++ b/core/translate/integrity_check.rs @@ -12,10 +12,14 @@ pub fn translate_integrity_check( program: &mut ProgramBuilder, ) -> crate::Result<()> { let mut root_pages = Vec::with_capacity(schema.tables.len() + schema.indexes.len()); + // Collect root pages to run integrity check on for table in schema.tables.values() { if let crate::schema::Table::BTree(table) = table.as_ref() { - root_pages.push(table.root_page); + if table.has_rowid { + root_pages.push(table.root_page); + } + if let Some(indexes) = schema.indexes.get(table.name.as_str()) { for index in indexes.iter() { if index.root_page > 0 { From de1d98ff00ade2a504955d9a8d1508ff065c0f7d Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 17:42:46 +0530 Subject: [PATCH 10/15] fmt --- core/translate/insert.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/translate/insert.rs b/core/translate/insert.rs index c783242d7a..bdb9c3dce4 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2756,7 +2756,7 @@ fn translate_without_rowid_insert( let pk_index = resolver .schema .get_indices(&table.name) - .find(|idx| idx.is_primary_key) + .find(|idx| idx.is_primary_key) .ok_or_else(|| { crate::error::LimboError::InternalError(format!( "WITHOUT ROWID table {} has no primary key index", From 4249b172034457216b19d2504a2a46960e220bcf Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Sun, 2 Nov 2025 18:25:00 +0530 Subject: [PATCH 11/15] stop re-adding prim key in secondary indexes,schema alrdy does it fmt --- core/translate/insert.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/core/translate/insert.rs b/core/translate/insert.rs index bdb9c3dce4..d5ee9169e9 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2899,7 +2899,7 @@ fn translate_without_rowid_insert( tracing::debug!("Processing secondary index '{}'", index.name); let secondary_key_cols = &index.columns; - let num_key_cols = secondary_key_cols.len() + pk_columns.len(); + let num_key_cols = secondary_key_cols.len(); let key_start_reg = program.alloc_registers(num_key_cols); let mut current_reg = key_start_reg; @@ -2914,16 +2914,6 @@ fn translate_without_rowid_insert( current_reg += 1; } - for pk_col in pk_columns { - let mapping = insertion.get_col_mapping_by_name(&pk_col.name).unwrap(); - program.emit_insn(Insn::Copy { - src_reg: mapping.register, - dst_reg: current_reg, - extra_amount: 0, - }); - current_reg += 1; - } - let record_reg = program.alloc_register(); program.emit_insn(Insn::MakeRecord { start_reg: key_start_reg, From 44f54558bf55c9368c688b368eec3483ae694601 Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Mon, 3 Nov 2025 08:24:11 +0530 Subject: [PATCH 12/15] add nkeycol to index, use for withoutrowid tables --- core/incremental/operator.rs | 1 + core/schema.rs | 6 ++++++ core/translate/compound_select.rs | 3 ++- core/translate/index.rs | 1 + core/translate/insert.rs | 25 +++++++++---------------- core/translate/main_loop.rs | 3 +++ core/translate/optimizer/mod.rs | 2 ++ core/translate/order_by.rs | 2 ++ core/translate/subquery.rs | 2 ++ 9 files changed, 28 insertions(+), 17 deletions(-) diff --git a/core/incremental/operator.rs b/core/incremental/operator.rs index 0914e41849..39dc617ec2 100644 --- a/core/incremental/operator.rs +++ b/core/incremental/operator.rs @@ -72,6 +72,7 @@ pub fn create_dbsp_state_index(root_page: i64) -> Index { where_clause: None, index_method: None, is_primary_key: false, + n_key_col: 3, } } diff --git a/core/schema.rs b/core/schema.rs index 6ff23445e7..d521d2fe10 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -2415,6 +2415,7 @@ pub struct Index { pub where_clause: Option>, pub index_method: Option>, pub is_primary_key: bool, + pub n_key_col: usize, } #[allow(dead_code)] @@ -2483,6 +2484,7 @@ impl Index { where_clause: None, is_primary_key: false, index_method: Some(descriptor), + n_key_col: columns.len(), }) } else { Ok(Index { @@ -2496,6 +2498,7 @@ impl Index { where_clause, index_method: None, is_primary_key: false, + n_key_col: columns.len(), }) } } @@ -2572,6 +2575,7 @@ impl Index { where_clause: None, index_method: None, is_primary_key: true, + n_key_col: column_count, }) } @@ -2600,6 +2604,7 @@ impl Index { }) .collect::>(); + let n_key_col = unique_cols.len(); Ok(Index { name: normalize_ident(index_name.as_str()), table_name: table.name.clone(), @@ -2611,6 +2616,7 @@ impl Index { where_clause: None, index_method: None, is_primary_key: false, + n_key_col, }) } diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 19f9e4918e..2d0028662c 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -437,7 +437,7 @@ fn create_dedupe_index( }; column.collation = collation; } - + let n_key_col = dedupe_columns.len(); let dedupe_index = Arc::new(Index { columns: dedupe_columns, name: "compound_dedupe".to_string(), @@ -449,6 +449,7 @@ fn create_dedupe_index( where_clause: None, index_method: None, is_primary_key: false, + n_key_col, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(dedupe_index.clone())); program.emit_insn(Insn::OpenEphemeral { diff --git a/core/translate/index.rs b/core/translate/index.rs index 83aa56eb27..5759d733de 100644 --- a/core/translate/index.rs +++ b/core/translate/index.rs @@ -150,6 +150,7 @@ pub fn translate_create_index( where_clause: where_clause.clone(), index_method: index_method.clone(), is_primary_key: false, + n_key_col: columns.len(), }); if !idx.validate_where_expr(table) { diff --git a/core/translate/insert.rs b/core/translate/insert.rs index d5ee9169e9..93fc037e66 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2765,9 +2765,10 @@ fn translate_without_rowid_insert( })?; tracing::debug!( - "Found primary key index '{}' with root page {}", + "Found WITHOUT ROWID primary key index '{}' with root page {} and {} key columns.", pk_index.name, - pk_index.root_page + pk_index.root_page, + pk_index.n_key_col ); let pk_cursor_id = program.alloc_cursor_index(None, pk_index)?; @@ -2807,8 +2808,7 @@ fn translate_without_rowid_insert( emit_notnulls(&mut program, &ctx, &insertion); - let pk_columns = &pk_index.columns; - let num_pk_cols = pk_columns.len(); + let num_pk_cols = pk_index.n_key_col; let pk_registers_start = program.alloc_registers(num_pk_cols); tracing::debug!( "Allocated registers {}-{} for PK uniqueness check", @@ -2816,7 +2816,7 @@ fn translate_without_rowid_insert( pk_registers_start + num_pk_cols - 1 ); - for (i, pk_col) in pk_columns.iter().enumerate() { + for (i, pk_col) in pk_index.columns.iter().take(num_pk_cols).enumerate() { let mapping = insertion.get_col_mapping_by_name(&pk_col.name).unwrap(); program.emit_insn(Insn::Copy { src_reg: mapping.register, @@ -2879,13 +2879,7 @@ fn translate_without_rowid_insert( tracing::debug!("Beginning update of secondary indices."); for index in resolver.schema.get_indices(table.name.as_str()) { - let is_pk = index.columns.len() == pk_index.columns.len() - && index - .columns - .iter() - .zip(pk_index.columns.iter()) - .all(|(a, b)| a.name == b.name); - if is_pk { + if index.is_primary_key { continue; } @@ -2898,13 +2892,12 @@ fn translate_without_rowid_insert( tracing::debug!("Processing secondary index '{}'", index.name); - let secondary_key_cols = &index.columns; - let num_key_cols = secondary_key_cols.len(); + + let num_key_cols = index.n_key_col; let key_start_reg = program.alloc_registers(num_key_cols); let mut current_reg = key_start_reg; - - for idx_col in secondary_key_cols { + for idx_col in index.columns.iter().take(num_key_cols) { let mapping = insertion.get_col_mapping_by_name(&idx_col.name).unwrap(); program.emit_insn(Insn::Copy { src_reg: mapping.register, diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 0f19564f12..323fdfb14a 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -109,6 +109,7 @@ pub fn init_distinct(program: &mut ProgramBuilder, plan: &SelectPlan) -> Result< where_clause: None, index_method: None, is_primary_key: false, + n_key_col: plan.result_columns.len(), }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); let ctx = DistinctCtx { @@ -188,6 +189,8 @@ pub fn init_loop( where_clause: None, index_method: None, is_primary_key: false, + n_key_col: group_by.as_ref().unwrap().exprs.len(), + }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); if group_by.is_none() { diff --git a/core/translate/optimizer/mod.rs b/core/translate/optimizer/mod.rs index dc1e8e9d5c..9335e6ccda 100644 --- a/core/translate/optimizer/mod.rs +++ b/core/translate/optimizer/mod.rs @@ -1292,6 +1292,7 @@ fn ephemeral_index_build( (None, None) => Ordering::Equal, } }); + let n_key_col = ephemeral_columns.len(); let ephemeral_index = Index { name: format!( "ephemeral_{}_{}", @@ -1310,6 +1311,7 @@ fn ephemeral_index_build( .is_some_and(|btree| btree.has_rowid), index_method: None, is_primary_key: false, + n_key_col, }; ephemeral_index diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 018cd77e89..1123982ab8 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -100,6 +100,7 @@ pub fn init_order_by( default: None, }) } + let n_key_col = index_columns.len(); let index = Arc::new(Index { name: index_name.clone(), table_name: String::new(), @@ -111,6 +112,7 @@ pub fn init_order_by( where_clause: None, index_method: None, is_primary_key: false, + n_key_col, }); program.alloc_cursor_id(CursorType::BTreeIndex(index)) } else { diff --git a/core/translate/subquery.rs b/core/translate/subquery.rs index 06ca1ce9cd..e5a9c75ed7 100644 --- a/core/translate/subquery.rs +++ b/core/translate/subquery.rs @@ -382,6 +382,7 @@ fn get_subquery_parser<'a>( )?; } + let n_key_col=columns.len(); let ephemeral_index = Arc::new(Index { columns, name: format!("ephemeral_index_where_sub_{subquery_id}"), @@ -393,6 +394,7 @@ fn get_subquery_parser<'a>( where_clause: None, index_method: None, is_primary_key: false, + n_key_col, }); let cursor_id = From fdc57971e82a5d66899c7d11f91dab7343b64aac Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Mon, 3 Nov 2025 20:08:08 +0530 Subject: [PATCH 13/15] store/order row acc to pkey --- core/incremental/operator.rs | 2 +- core/schema.rs | 10 ++--- core/translate/compound_select.rs | 4 +- core/translate/index.rs | 2 +- core/translate/insert.rs | 69 ++++++++++++++++++++++++++----- core/translate/main_loop.rs | 3 +- core/translate/optimizer/mod.rs | 4 +- core/translate/order_by.rs | 4 +- core/translate/subquery.rs | 4 +- 9 files changed, 75 insertions(+), 27 deletions(-) diff --git a/core/incremental/operator.rs b/core/incremental/operator.rs index 39dc617ec2..d22c20e1b0 100644 --- a/core/incremental/operator.rs +++ b/core/incremental/operator.rs @@ -72,7 +72,7 @@ pub fn create_dbsp_state_index(root_page: i64) -> Index { where_clause: None, index_method: None, is_primary_key: false, - n_key_col: 3, + n_key_col: 3, } } diff --git a/core/schema.rs b/core/schema.rs index d521d2fe10..966b339ccd 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -2415,7 +2415,7 @@ pub struct Index { pub where_clause: Option>, pub index_method: Option>, pub is_primary_key: bool, - pub n_key_col: usize, + pub n_key_col: usize, } #[allow(dead_code)] @@ -2484,7 +2484,7 @@ impl Index { where_clause: None, is_primary_key: false, index_method: Some(descriptor), - n_key_col: columns.len(), + n_key_col: columns.len(), }) } else { Ok(Index { @@ -2498,7 +2498,7 @@ impl Index { where_clause, index_method: None, is_primary_key: false, - n_key_col: columns.len(), + n_key_col: columns.len(), }) } } @@ -2604,7 +2604,7 @@ impl Index { }) .collect::>(); - let n_key_col = unique_cols.len(); + let n_key_col = unique_cols.len(); Ok(Index { name: normalize_ident(index_name.as_str()), table_name: table.name.clone(), @@ -2616,7 +2616,7 @@ impl Index { where_clause: None, index_method: None, is_primary_key: false, - n_key_col, + n_key_col, }) } diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 2d0028662c..ee5cb248bf 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -437,7 +437,7 @@ fn create_dedupe_index( }; column.collation = collation; } - let n_key_col = dedupe_columns.len(); + let n_key_col = dedupe_columns.len(); let dedupe_index = Arc::new(Index { columns: dedupe_columns, name: "compound_dedupe".to_string(), @@ -449,7 +449,7 @@ fn create_dedupe_index( where_clause: None, index_method: None, is_primary_key: false, - n_key_col, + n_key_col, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(dedupe_index.clone())); program.emit_insn(Insn::OpenEphemeral { diff --git a/core/translate/index.rs b/core/translate/index.rs index 5759d733de..f19373dc8e 100644 --- a/core/translate/index.rs +++ b/core/translate/index.rs @@ -150,7 +150,7 @@ pub fn translate_create_index( where_clause: where_clause.clone(), index_method: index_method.clone(), is_primary_key: false, - n_key_col: columns.len(), + n_key_col: columns.len(), }); if !idx.validate_where_expr(table) { diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 93fc037e66..0d75b85e53 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2842,32 +2842,82 @@ fn translate_without_rowid_insert( description: format_unique_violation_desc(&table.name, pk_index), }); + program.preassign_label_to_next_insn(ok_to_insert_label); tracing::debug!("Uniqueness check passed. Continuing with insertion."); + + let physically_ordered_cols: Vec<&Column> = { + let mut ordered = Vec::with_capacity(table.columns.len()); + let mut pk_cols_added = std::collections::HashSet::new(); + + for pk_col_def in pk_index.columns.iter().take(pk_index.n_key_col) { + let (_, col) = table.get_column(&pk_col_def.name).unwrap(); + ordered.push(col); + pk_cols_added.insert(col.name.as_ref().unwrap().as_str()); + tracing::trace!( + " Physical col {}: {}", + ordered.len() - 1, + col.name.as_ref().unwrap() + ); + } + + for col in table.columns.iter() { + if !pk_cols_added.contains(col.name.as_ref().unwrap().as_str()) { + ordered.push(col); + tracing::trace!( + " Physical col {}: {}", + ordered.len() - 1, + col.name.as_ref().unwrap() + ); + } + } + assert_eq!( + ordered.len(), + table.columns.len(), + "Physical column ordering failed: count mismatch." + ); + ordered + }; + + let reordered_start_reg = program.alloc_registers(insertion.col_mappings.len()); + tracing::debug!( + "Reordering columns into physical order. New registers start at {}", + reordered_start_reg + ); + + for (i, physical_col) in physically_ordered_cols.iter().enumerate() { + let logical_mapping = insertion + .get_col_mapping_by_name(physical_col.name.as_ref().unwrap()) + .unwrap(); + program.emit_insn(Insn::Copy { + src_reg: logical_mapping.register, + dst_reg: reordered_start_reg + i, + extra_amount: 0, + }); + } + let full_record_reg = program.alloc_register(); - let affinity_str = insertion - .col_mappings + let affinity_str = physically_ordered_cols .iter() - .map(|col_mapping| col_mapping.column.affinity().aff_mask()) + .map(|col| col.affinity().aff_mask()) .collect::(); + program.emit_insn(Insn::MakeRecord { - start_reg: insertion.first_col_register(), + start_reg: reordered_start_reg, count: insertion.col_mappings.len(), dest_reg: full_record_reg, index_name: Some(pk_index.name.clone()), affinity_str: Some(affinity_str), }); - tracing::debug!( - "Emitted OP_MakeRecord for full row into register {}", - full_record_reg - ); + + program.emit_insn(Insn::IdxInsert { cursor_id: pk_cursor_id, record_reg: full_record_reg, - unpacked_start: Some(insertion.first_col_register()), + unpacked_start: Some(reordered_start_reg), unpacked_count: Some(insertion.col_mappings.len() as u16), flags: IdxInsertFlags::new().nchange(true), }); @@ -2892,7 +2942,6 @@ fn translate_without_rowid_insert( tracing::debug!("Processing secondary index '{}'", index.name); - let num_key_cols = index.n_key_col; let key_start_reg = program.alloc_registers(num_key_cols); diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 323fdfb14a..5641258229 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -189,8 +189,7 @@ pub fn init_loop( where_clause: None, index_method: None, is_primary_key: false, - n_key_col: group_by.as_ref().unwrap().exprs.len(), - + n_key_col: group_by.as_ref().unwrap().exprs.len(), }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); if group_by.is_none() { diff --git a/core/translate/optimizer/mod.rs b/core/translate/optimizer/mod.rs index 9335e6ccda..ad6315668d 100644 --- a/core/translate/optimizer/mod.rs +++ b/core/translate/optimizer/mod.rs @@ -1292,7 +1292,7 @@ fn ephemeral_index_build( (None, None) => Ordering::Equal, } }); - let n_key_col = ephemeral_columns.len(); + let n_key_col = ephemeral_columns.len(); let ephemeral_index = Index { name: format!( "ephemeral_{}_{}", @@ -1311,7 +1311,7 @@ fn ephemeral_index_build( .is_some_and(|btree| btree.has_rowid), index_method: None, is_primary_key: false, - n_key_col, + n_key_col, }; ephemeral_index diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 1123982ab8..35e94ba33a 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -100,7 +100,7 @@ pub fn init_order_by( default: None, }) } - let n_key_col = index_columns.len(); + let n_key_col = index_columns.len(); let index = Arc::new(Index { name: index_name.clone(), table_name: String::new(), @@ -112,7 +112,7 @@ pub fn init_order_by( where_clause: None, index_method: None, is_primary_key: false, - n_key_col, + n_key_col, }); program.alloc_cursor_id(CursorType::BTreeIndex(index)) } else { diff --git a/core/translate/subquery.rs b/core/translate/subquery.rs index e5a9c75ed7..99b97fa841 100644 --- a/core/translate/subquery.rs +++ b/core/translate/subquery.rs @@ -382,7 +382,7 @@ fn get_subquery_parser<'a>( )?; } - let n_key_col=columns.len(); + let n_key_col = columns.len(); let ephemeral_index = Arc::new(Index { columns, name: format!("ephemeral_index_where_sub_{subquery_id}"), @@ -394,7 +394,7 @@ fn get_subquery_parser<'a>( where_clause: None, index_method: None, is_primary_key: false, - n_key_col, + n_key_col, }); let cursor_id = From 3ad7f7495a31f91e380e462e63d70b16bb416fd3 Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Mon, 3 Nov 2025 22:14:05 +0530 Subject: [PATCH 14/15] fix panic in aggr queries for n_key_col, return 0 if no grpby --- core/translate/insert.rs | 5 ----- core/translate/main_loop.rs | 6 +++++- core/translate/schema.rs | 1 - 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 0d75b85e53..bece7aa57f 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2842,11 +2842,9 @@ fn translate_without_rowid_insert( description: format_unique_violation_desc(&table.name, pk_index), }); - program.preassign_label_to_next_insn(ok_to_insert_label); tracing::debug!("Uniqueness check passed. Continuing with insertion."); - let physically_ordered_cols: Vec<&Column> = { let mut ordered = Vec::with_capacity(table.columns.len()); let mut pk_cols_added = std::collections::HashSet::new(); @@ -2903,7 +2901,6 @@ fn translate_without_rowid_insert( .map(|col| col.affinity().aff_mask()) .collect::(); - program.emit_insn(Insn::MakeRecord { start_reg: reordered_start_reg, count: insertion.col_mappings.len(), @@ -2912,8 +2909,6 @@ fn translate_without_rowid_insert( affinity_str: Some(affinity_str), }); - - program.emit_insn(Insn::IdxInsert { cursor_id: pk_cursor_id, record_reg: full_record_reg, diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 5641258229..d69d77dab1 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -189,7 +189,11 @@ pub fn init_loop( where_clause: None, index_method: None, is_primary_key: false, - n_key_col: group_by.as_ref().unwrap().exprs.len(), + n_key_col: if let Some(gb) = group_by.as_ref() { + gb.exprs.len() + } else { + 0 + }, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); if group_by.is_none() { diff --git a/core/translate/schema.rs b/core/translate/schema.rs index a297b74fe4..575de0c65e 100644 --- a/core/translate/schema.rs +++ b/core/translate/schema.rs @@ -452,7 +452,6 @@ fn collect_autoindexes( }; let needs_index = if us.is_primary_key { - !(is_without_rowid || (col.primary_key() && col.is_rowid_alias())) } else { // UNIQUE single needs an index From 9f56ca40790559a9f9092bde9039f3e2e4bca30e Mon Sep 17 00:00:00 2001 From: Pavan-Nambi Date: Tue, 4 Nov 2025 20:15:28 +0530 Subject: [PATCH 15/15] sec index on without rowid tables shall have prim key cols fully.. --- core/schema.rs | 28 +++++++++++++++++++++++++++- core/translate/insert.rs | 23 ++++++++++++++--------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/core/schema.rs b/core/schema.rs index 966b339ccd..38cb63a3c9 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -2454,7 +2454,33 @@ impl Index { .. })) => { let index_name = normalize_ident(idx_name.name.as_str()); - let index_columns = resolve_sorted_columns(table, &columns)?; + let mut index_columns = resolve_sorted_columns(table, &columns)?; + + if !table.has_rowid { + let existing_index_cols: HashSet = + index_columns.iter().map(|c| c.name.clone()).collect(); + + for (pk_col_name, _order) in &table.primary_key_columns { + if !existing_index_cols.contains(pk_col_name.as_str()) { + let (pos_in_table, column) = + table.get_column(pk_col_name).ok_or_else(|| { + LimboError::InternalError(format!( + "Could not find PK column '{}' in table '{}'", + pk_col_name, table.name + )) + })?; + + index_columns.push(IndexColumn { + name: pk_col_name.clone(), + order: SortOrder::Asc, + pos_in_table, + collation: column.collation_opt(), + default: column.default.clone(), + }); + } + } + } + if let Some(using) = using { if where_clause.is_some() { bail_parse_error!("custom index module do not support partial indices"); diff --git a/core/translate/insert.rs b/core/translate/insert.rs index bece7aa57f..34cff50429 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -2937,24 +2937,29 @@ fn translate_without_rowid_insert( tracing::debug!("Processing secondary index '{}'", index.name); - let num_key_cols = index.n_key_col; - let key_start_reg = program.alloc_registers(num_key_cols); + let num_total_cols = index.columns.len(); + let key_start_reg = program.alloc_registers(num_total_cols); - let mut current_reg = key_start_reg; - for idx_col in index.columns.iter().take(num_key_cols) { - let mapping = insertion.get_col_mapping_by_name(&idx_col.name).unwrap(); + for (i, idx_col) in index.columns.iter().enumerate() { + let mapping = insertion + .get_col_mapping_by_name(&idx_col.name) + .unwrap_or_else(|| { + panic!( + "Could not find column mapping for '{}' in index '{}'", + idx_col.name, index.name + ) + }); program.emit_insn(Insn::Copy { src_reg: mapping.register, - dst_reg: current_reg, + dst_reg: key_start_reg + i, extra_amount: 0, }); - current_reg += 1; } let record_reg = program.alloc_register(); program.emit_insn(Insn::MakeRecord { start_reg: key_start_reg, - count: num_key_cols, + count: num_total_cols, dest_reg: record_reg, index_name: Some(index.name.clone()), affinity_str: None, @@ -2969,7 +2974,7 @@ fn translate_without_rowid_insert( cursor_id: idx_cursor_id, record_reg, unpacked_start: Some(key_start_reg), - unpacked_count: Some(num_key_cols as u16), + unpacked_count: Some(num_total_cols as u16), flags: IdxInsertFlags::new().nchange(true), }); tracing::debug!("Emitted OP_IdxInsert for secondary index '{}'", index.name);