Without rowid

core/incremental/operator.rs

@@ -71,6 +71,8 @@ pub fn create_dbsp_state_index(root_page: i64) -> Index {
         has_rowid: true,
         where_clause: None,
         index_method: None,
+        is_primary_key: false,
+        n_key_col: 3,
     }
 }
 

core/schema.rs

@@ -751,7 +751,28 @@ impl Schema {
                         }
                     }
 
+                    let has_rowid = table.has_rowid;
+                    let table_name_for_index = table.name.clone();
+                    let root_page_for_index = table.root_page;
+
                     self.add_btree_table(Arc::new(table))?;
+
+                    if !has_rowid {
+                        tracing::debug!(
+                            "WITHOUT ROWID table '{}' found. Synthesizing primary key index.",
+                            &table_name_for_index
+                        );
+                        let table_ref = self.get_btree_table(&table_name_for_index).unwrap();
+                        let pk_index = Index::automatic_from_primary_key(
+                            &table_ref,
+                            (
+                                format!("sqlite_autoindex_{}_1", &table_name_for_index),
+                                root_page_for_index,
+                            ),
+                            table_ref.primary_key_columns.len(),
+                        )?;
+                        self.add_index(Arc::new(pk_index))?;
+                    }
                 }
             }
             "index" => {
@@ -2393,6 +2414,8 @@ pub struct Index {
     pub has_rowid: bool,
     pub where_clause: Option<Box<Expr>>,
     pub index_method: Option<Arc<dyn IndexMethodAttachment>>,
+    pub is_primary_key: bool,
+    pub n_key_col: usize,
 }
 
 #[allow(dead_code)]
@@ -2431,7 +2454,33 @@ impl Index {
                 ..
             })) => {
                 let index_name = normalize_ident(idx_name.name.as_str());
-                let index_columns = resolve_sorted_columns(table, &columns)?;
+                let mut index_columns = resolve_sorted_columns(table, &columns)?;
+
+                if !table.has_rowid {
+                    let existing_index_cols: HashSet<String> =
+                        index_columns.iter().map(|c| c.name.clone()).collect();
+
+                    for (pk_col_name, _order) in &table.primary_key_columns {
+                        if !existing_index_cols.contains(pk_col_name.as_str()) {
+                            let (pos_in_table, column) =
+                                table.get_column(pk_col_name).ok_or_else(|| {
+                                    LimboError::InternalError(format!(
+                                        "Could not find PK column '{}' in table '{}'",
+                                        pk_col_name, table.name
+                                    ))
+                                })?;
+
+                            index_columns.push(IndexColumn {
+                                name: pk_col_name.clone(),
+                                order: SortOrder::Asc,
+                                pos_in_table,
+                                collation: column.collation_opt(),
+                                default: column.default.clone(),
+                            });
+                        }
+                    }
+                }
+
                 if let Some(using) = using {
                     if where_clause.is_some() {
                         bail_parse_error!("custom index module do not support partial indices");
@@ -2459,7 +2508,9 @@ impl Index {
                         ephemeral: false,
                         has_rowid: table.has_rowid,
                         where_clause: None,
+                        is_primary_key: false,
                         index_method: Some(descriptor),
+                        n_key_col: columns.len(),
                     })
                 } else {
                     Ok(Index {
@@ -2472,6 +2523,8 @@ impl Index {
                         has_rowid: table.has_rowid,
                         where_clause,
                         index_method: None,
+                        is_primary_key: false,
+                        n_key_col: columns.len(),
                     })
                 }
             }
@@ -2496,36 +2549,59 @@ impl Index {
         assert!(has_primary_key_index);
         let (index_name, root_page) = auto_index;
 
-        let mut primary_keys = Vec::with_capacity(column_count);
+        let mut index_columns = Vec::new();
+        let mut key_column_positions = std::collections::HashSet::new();
+
         for (col_name, order) in table.primary_key_columns.iter() {
-            let Some((pos_in_table, _)) = table.get_column(col_name) else {
+            let Some((pos_in_table, column)) = table.get_column(col_name) else {
                 return Err(crate::LimboError::ParseError(format!(
                     "Column {} not found in table {}",
                     col_name, table.name
                 )));
             };
-            let (_, column) = table.get_column(col_name).unwrap();
-            primary_keys.push(IndexColumn {
+            index_columns.push(IndexColumn {
                 name: normalize_ident(col_name),
                 order: *order,
                 pos_in_table,
                 collation: column.collation_opt(),
                 default: column.default.clone(),
             });
+            key_column_positions.insert(pos_in_table);
         }
 
-        assert!(primary_keys.len() == column_count);
+        assert!(
+            index_columns.len() == column_count,
+            "Mismatch in primary key column count"
+        );
+
+        if !table.has_rowid {
+            for (pos_in_table, column) in table.columns.iter().enumerate() {
+                // TODO: when we support generated cols look at this
+
+                if !key_column_positions.contains(&pos_in_table) {
+                    index_columns.push(IndexColumn {
+                        name: normalize_ident(column.name.as_ref().unwrap()),
+                        order: SortOrder::Asc,
+                        pos_in_table,
+                        collation: None,
+                        default: column.default.clone(),
+                    });
+                }
+            }
+        }
 
         Ok(Index {
             name: normalize_ident(index_name.as_str()),
             table_name: table.name.clone(),
             root_page,
-            columns: primary_keys,
+            columns: index_columns,
             unique: true,
             ephemeral: false,
             has_rowid: table.has_rowid,
             where_clause: None,
             index_method: None,
+            is_primary_key: true,
+            n_key_col: column_count,
         })
     }
 
@@ -2554,6 +2630,7 @@ impl Index {
             })
             .collect::<Vec<_>>();
 
+        let n_key_col = unique_cols.len();
         Ok(Index {
             name: normalize_ident(index_name.as_str()),
             table_name: table.name.clone(),
@@ -2564,6 +2641,8 @@ impl Index {
             has_rowid: table.has_rowid,
             where_clause: None,
             index_method: None,
+            is_primary_key: false,
+            n_key_col,
         })
     }
 

core/storage/btree.rs

@@ -5044,6 +5044,26 @@ impl CursorTrait for BTreeCursor {
 
     #[instrument(skip_all, level = Level::DEBUG)]
     fn insert(&mut self, key: &BTreeKey) -> Result<IOResult<()>> {
+        // remove this after getting sanity, before merging
+
+        match key {
+            BTreeKey::TableRowId((rowid, record)) => {
+                tracing::debug!(
+                    "BTreeCursor::insert: root_page={}, key_type=TableRowId, rowid={}, record_size={}",
+                    self.root_page,
+                    rowid,
+                    record.map_or(0, |r| r.get_payload().len())
+                );
+            }
+            BTreeKey::IndexKey(record) => {
+                tracing::debug!(
+                    "BTreeCursor::insert: root_page={}, key_type=IndexKey, record_size={}",
+                    self.root_page,
+                    record.get_payload().len()
+                );
+            }
+        }
+
         tracing::debug!(valid_state = ?self.valid_state, cursor_state = ?self.state, is_write_in_progress = self.is_write_in_progress());
         return_if_io!(self.insert_into_page(key));
         if key.maybe_rowid().is_some() {
@@ -8558,6 +8578,7 @@ mod tests {
                 ephemeral: false,
                 has_rowid: false,
                 index_method: None,
+                is_primary_key: false,
             };
             let num_columns = index_def.columns.len();
             let mut cursor =
@@ -8718,6 +8739,7 @@ mod tests {
                 ephemeral: false,
                 has_rowid: false,
                 index_method: None,
+                is_primary_key: false,
             };
             let mut cursor = BTreeCursor::new_index(pager.clone(), index_root_page, &index_def, 1);
 

core/translate/compound_select.rs

@@ -437,7 +437,7 @@ fn create_dedupe_index(
         };
         column.collation = collation;
     }
-
+    let n_key_col = dedupe_columns.len();
     let dedupe_index = Arc::new(Index {
         columns: dedupe_columns,
         name: "compound_dedupe".to_string(),
@@ -448,6 +448,8 @@ fn create_dedupe_index(
         has_rowid: false,
         where_clause: None,
         index_method: None,
+        is_primary_key: false,
+        n_key_col,
     });
     let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(dedupe_index.clone()));
     program.emit_insn(Insn::OpenEphemeral {

core/translate/index.rs

@@ -149,6 +149,8 @@ pub fn translate_create_index(
         // before translating, and it cannot reference a table alias
         where_clause: where_clause.clone(),
         index_method: index_method.clone(),
+        is_primary_key: false,
+        n_key_col: columns.len(),
     });
 
     if !idx.validate_where_expr(table) {