From fc5e42bb94e0fa5bc0d94b0e96a41c907db84d76 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 13 Nov 2024 12:54:09 +0200 Subject: [PATCH 1/7] `CREATE TABLE` support --- core/translate/mod.rs | 208 +++++++++++++++++++++++++++++++++++++++++- core/vdbe/builder.rs | 4 + core/vdbe/explain.rs | 27 ++++++ core/vdbe/mod.rs | 38 ++++++++ sqlite3/src/lib.rs | 2 +- 5 files changed, 277 insertions(+), 2 deletions(-) diff --git a/core/translate/mod.rs b/core/translate/mod.rs index dc77a00b..17ed9dec 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -42,7 +42,24 @@ pub fn translate( ast::Stmt::Begin(_, _) => bail_parse_error!("BEGIN not supported yet"), ast::Stmt::Commit(_) => bail_parse_error!("COMMIT not supported yet"), ast::Stmt::CreateIndex { .. } => bail_parse_error!("CREATE INDEX not supported yet"), - ast::Stmt::CreateTable { .. } => bail_parse_error!("CREATE TABLE not supported yet"), + ast::Stmt::CreateTable { + temporary, + if_not_exists, + tbl_name, + body, + } => { + if temporary { + bail_parse_error!("TEMPORARY table not supported yet"); + } + translate_create_table( + tbl_name, + body, + if_not_exists, + database_header, + connection, + schema, + ) + } ast::Stmt::CreateTrigger { .. } => bail_parse_error!("CREATE TRIGGER not supported yet"), ast::Stmt::CreateView { .. } => bail_parse_error!("CREATE VIEW not supported yet"), ast::Stmt::CreateVirtualTable { .. } => { @@ -85,6 +102,195 @@ pub fn translate( } } +/* Example: + +sqlite> EXPLAIN CREATE TABLE users (id INT, email TEXT);; +addr opcode p1 p2 p3 p4 p5 comment +---- ------------- ---- ---- ---- ------------- -- ------------- +0 Init 0 30 0 0 Start at 30 +1 ReadCookie 0 3 2 0 +2 If 3 5 0 0 +3 SetCookie 0 2 4 0 +4 SetCookie 0 5 1 0 +5 CreateBtree 0 2 1 0 r[2]=root iDb=0 flags=1 +6 OpenWrite 0 1 0 5 0 root=1 iDb=0 +7 NewRowid 0 1 0 0 r[1]=rowid +8 Blob 6 3 0 0 r[3]= (len=6) +9 Insert 0 3 1 8 intkey=r[1] data=r[3] +10 Close 0 0 0 0 +11 Close 0 0 0 0 +12 Null 0 4 5 0 r[4..5]=NULL +13 Noop 2 0 4 0 +14 OpenWrite 1 1 0 5 0 root=1 iDb=0; sqlite_master +15 SeekRowid 1 17 1 0 intkey=r[1] +16 Rowid 1 5 0 0 r[5]= rowid of 1 +17 IsNull 5 26 0 0 if r[5]==NULL goto 26 +18 String8 0 6 0 table 0 r[6]='table' +19 String8 0 7 0 users 0 r[7]='users' +20 String8 0 8 0 users 0 r[8]='users' +21 Copy 2 9 0 0 r[9]=r[2] +22 String8 0 10 0 CREATE TABLE users (id INT, email TEXT) 0 r[10]='CREATE TABLE users (id INT, email TEXT)' +23 MakeRecord 6 5 4 BBBDB 0 r[4]=mkrec(r[6..10]) +24 Delete 1 68 5 0 +25 Insert 1 4 5 0 intkey=r[5] data=r[4] +26 SetCookie 0 1 1 0 +27 ParseSchema 0 0 0 tbl_name='users' AND type!='trigger' 0 +28 SqlExec 1 0 0 PRAGMA "main".integrity_check('users') 0 +29 Halt 0 0 0 0 +30 Transaction 0 1 0 0 1 usesStmtJournal=1 +31 Goto 0 1 0 0 + +*/ +fn translate_create_table( + tbl_name: ast::QualifiedName, + body: ast::CreateTableBody, + if_not_exists: bool, + database_header: Rc>, + connection: Weak, + schema: &Schema, +) -> Result { + let mut program = ProgramBuilder::new(); + let parse_schema_label = program.allocate_label(); + let init_label = program.allocate_label(); + program.emit_insn_with_label_dependency( + Insn::Init { + target_pc: init_label, + }, + init_label, + ); + let start_offset = program.offset(); + // TODO: ReadCookie + // TODO: If + // TODO: SetCookie + // TODO: SetCookie + let root_reg = program.alloc_register(); + program.emit_insn(Insn::CreateBtree { + db: 0, + root: root_reg, + flags: 1, + }); + let table_id = "sqlite_schema".to_string(); + let table = schema.get_table(&table_id).unwrap(); + let table = crate::schema::Table::BTree(table.clone()); + let sqlite_schema_cursor_id = + program.alloc_cursor_id(Some(table_id.to_owned()), Some(table.to_owned())); + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: sqlite_schema_cursor_id, + root_page: 1, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + let rowid_reg = program.alloc_register(); + program.emit_insn(Insn::NewRowid { + cursor: sqlite_schema_cursor_id, + rowid_reg, + prev_largest_reg: 0, + }); + let blob_reg = program.alloc_register(); + program.emit_insn(Insn::Blob { + value: vec![0, 0, 0, 0, 0, 0], + dest: blob_reg, + }); + program.emit_insn(Insn::InsertAsync { + cursor: sqlite_schema_cursor_id, + key_reg: rowid_reg, + record_reg: blob_reg, + flag: 0, + }); + program.emit_insn(Insn::InsertAwait { + cursor_id: sqlite_schema_cursor_id, + }); + program.emit_insn(Insn::Close { + cursor_id: sqlite_schema_cursor_id, + }); + program.emit_insn(Insn::Close { + cursor_id: sqlite_schema_cursor_id, + }); + let null_reg_1 = program.alloc_register(); + let null_reg_2 = program.alloc_register(); + program.emit_insn(Insn::Null { + dest: null_reg_1, + dest_end: Some(null_reg_2), + }); + // TODO: Noop + let sqlite_schema_cursor_id = program.alloc_cursor_id(Some(table_id), Some(table)); + program.emit_insn(Insn::OpenWriteAsync { + cursor_id: sqlite_schema_cursor_id, + root_page: 1, + }); + program.emit_insn(Insn::OpenWriteAwait {}); + let seek_failed_label = program.allocate_label(); + program.emit_insn_with_label_dependency( + Insn::SeekRowid { + cursor_id: sqlite_schema_cursor_id, + src_reg: rowid_reg, + target_pc: seek_failed_label, + }, + seek_failed_label, + ); + let rowid_reg_2 = program.alloc_register(); + program.emit_insn(Insn::RowId { + cursor_id: sqlite_schema_cursor_id, + dest: rowid_reg_2, + }); + program.resolve_label(seek_failed_label, program.offset()); + program.emit_insn_with_label_dependency( + Insn::IsNull { + src: rowid_reg, + target_pc: parse_schema_label, + }, + parse_schema_label, + ); + let type_reg = program.alloc_register(); + program.emit_insn(Insn::String8 { + value: "table".to_string(), + dest: type_reg, + }); + let name_reg = program.alloc_register(); + program.emit_insn(Insn::String8 { + value: tbl_name.name.0.to_string(), + dest: name_reg, + }); + let tbl_name_reg = program.alloc_register(); + program.emit_insn(Insn::String8 { + value: tbl_name.name.0.to_string(), + dest: tbl_name_reg, + }); + let rootpage_reg = program.alloc_register(); + program.emit_insn(Insn::Copy { + src_reg: root_reg, + dst_reg: rootpage_reg, + amount: 1, + }); + let sql_reg = program.alloc_register(); + program.emit_insn(Insn::String8 { + value: "TODO".to_string(), + dest: sql_reg, + }); + let record_reg = program.alloc_register(); + program.emit_insn(Insn::MakeRecord { + start_reg: type_reg, + count: 4, + dest_reg: record_reg, + }); + // TODO: Delete + // TODO: Insert + program.resolve_label(parse_schema_label, program.offset()); + // TODO: SetCookie + // TODO: ParseSchema + // TODO: SqlExec + program.emit_insn(Insn::Halt { + err_code: 0, + description: String::new(), + }); + program.resolve_label(init_label, program.offset()); + program.emit_insn(Insn::Transaction { write: true }); + program.emit_constant_insns(); + program.emit_insn(Insn::Goto { + target_pc: start_offset, + }); + Ok(program.build(database_header, connection)) +} + fn translate_pragma( name: &ast::QualifiedName, body: Option, diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index ee2bdb61..e9d8ac95 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -319,6 +319,10 @@ impl ProgramBuilder { assert!(*target_pc < 0); *target_pc = to_offset; } + Insn::IsNull { src, target_pc } => { + assert!(*target_pc < 0); + *target_pc = to_offset; + } _ => { todo!("missing resolve_label for {:?}", insn); } diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index b437b9d5..ae6f4ec7 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -866,6 +866,33 @@ pub fn insn_to_str( 0, format!("r[{}]=r[{}]", dst_reg, src_reg), ), + Insn::CreateBtree { db, root, flags } => ( + "CreateBtree", + *db as i32, + *root as i32, + *flags as i32, + OwnedValue::Text(Rc::new("".to_string())), + 0, + format!("r[{}]=root iDb={} flags={}", root, db, flags), + ), + Insn::Close { cursor_id } => ( + "Close", + *cursor_id as i32, + 0, + 0, + OwnedValue::Text(Rc::new("".to_string())), + 0, + "".to_string(), + ), + Insn::IsNull { src, target_pc } => ( + "IsNull", + *src as i32, + *target_pc as i32, + 0, + OwnedValue::Text(Rc::new("".to_string())), + 0, + format!("if (r[{}]==NULL) goto {}", src, target_pc), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 32216514..3d15c549 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -469,6 +469,30 @@ pub enum Insn { dst_reg: usize, amount: usize, // 0 amount means we include src_reg, dst_reg..=dst_reg+amount = src_reg..=src_reg+amount }, + + /// Allocate a new b-tree. + CreateBtree { + /// Allocate b-tree in main database if zero or in temp database if non-zero (P1). + db: usize, + /// The root page of the new b-tree (P2). + root: usize, + /// Flags (P3). + flags: usize, + }, + + /// Close a cursor. + Close { + cursor_id: CursorID, + }, + + /// Check if the register is null. + IsNull { + /// Source register (P1). + src: usize, + + /// Jump to this PC if the register is null (P2). + target_pc: BranchOffset, + }, } // Index of insn in list of insns @@ -2087,6 +2111,20 @@ impl Program { } state.pc += 1; } + Insn::CreateBtree { .. } => { + todo!(); + } + Insn::Close { cursor_id } => { + cursors.remove(cursor_id); + state.pc += 1; + } + Insn::IsNull { src, target_pc } => { + if matches!(state.registers[*src], OwnedValue::Null) { + state.pc = *target_pc; + } else { + state.pc += 1; + } + } } } } diff --git a/sqlite3/src/lib.rs b/sqlite3/src/lib.rs index 67c975fb..ac37d8ea 100644 --- a/sqlite3/src/lib.rs +++ b/sqlite3/src/lib.rs @@ -885,7 +885,7 @@ fn sqlite3_errstr_impl(rc: i32) -> *const std::ffi::c_char { "datatype mismatch", // SQLITE_MISMATCH "bad parameter or other API misuse", // SQLITE_MISUSE #[cfg(feature = "lfs")] - "", // SQLITE_NOLFS + "", // SQLITE_NOLFS #[cfg(not(feature = "lfs"))] "large file support is disabled", // SQLITE_NOLFS "authorization denied", // SQLITE_AUTH From 66e20df309faa9ddaffbc70b1254fbc3dbbd4e52 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Sat, 16 Nov 2024 16:23:06 +0100 Subject: [PATCH 2/7] fix some offset usages in btree --- core/storage/btree.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 18ceb5f9..6383d065 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -648,7 +648,7 @@ impl BTreeCursor { pointer_area_pc_by_idx + 2, ); } - page.write_u16(pointer_area_pc_by_idx, pc); + page.write_u16(pointer_area_pc_by_idx - page.offset, pc); // update first byte of content area page.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, pc); @@ -1128,17 +1128,13 @@ impl BTreeCursor { if gap + 2 + amount > top { // defragment self.defragment_page(page_ref, RefCell::borrow(&self.database_header)); - let buf = page_ref.as_ptr(); - top = u16::from_be_bytes([buf[5], buf[6]]) as usize; + top = page_ref.read_u16(BTREE_HEADER_OFFSET_CELL_CONTENT) as usize; } let db_header = RefCell::borrow(&self.database_header); top -= amount; - { - let buf = page_ref.as_ptr(); - buf[5..7].copy_from_slice(&(top as u16).to_be_bytes()); - } + page_ref.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, top as u16); let usable_space = (db_header.page_size - db_header.unused_space as u16) as usize; assert!(top + amount <= usable_space); @@ -1358,6 +1354,7 @@ impl BTreeCursor { let id = page.id as u32; let contents = page.contents.as_mut().unwrap(); + // TODO: take into account offset here? let buf = contents.as_ptr(); let as_bytes = id.to_be_bytes(); // update pointer to new overflow page From 090615b28992574ec0e1215c0a1b97a5929afebd Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Sat, 16 Nov 2024 16:24:28 +0100 Subject: [PATCH 3/7] create btree table + parse schema --- core/lib.rs | 66 ++++++++++--------------------------- core/pseudo.rs | 4 +++ core/storage/btree.rs | 14 ++++++++ core/translate/mod.rs | 75 +++++++++++++------------------------------ core/types.rs | 2 ++ core/util.rs | 49 ++++++++++++++++++++++++++++ core/vdbe/explain.rs | 9 ++++++ core/vdbe/mod.rs | 40 +++++++++++++++++++++-- core/vdbe/sorter.rs | 4 +++ 9 files changed, 158 insertions(+), 105 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 9bc1bddc..cdb35549 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -29,6 +29,7 @@ use storage::database::FileStorage; use storage::pager::allocate_page; use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}; pub use storage::wal::WalFile; +use util::parse_schema_rows; use translate::optimizer::optimize_plan; use translate::planner::prepare_select_plan; @@ -59,7 +60,7 @@ enum TransactionState { pub struct Database { pager: Rc, - schema: Rc, + schema: Rc>, header: Rc>, transaction_state: RefCell, } @@ -98,7 +99,7 @@ impl Database { wal, io.clone(), )?); - let bootstrap_schema = Rc::new(Schema::new()); + let bootstrap_schema = Rc::new(RefCell::new(Schema::new())); let conn = Rc::new(Connection { pager: pager.clone(), schema: bootstrap_schema.clone(), @@ -107,46 +108,8 @@ impl Database { }); let mut schema = Schema::new(); let rows = conn.query("SELECT * FROM sqlite_schema")?; - if let Some(mut rows) = rows { - loop { - match rows.next_row()? { - RowResult::Row(row) => { - let ty = row.get::<&str>(0)?; - if ty != "table" && ty != "index" { - continue; - } - match ty { - "table" => { - let root_page: i64 = row.get::(3)?; - let sql: &str = row.get::<&str>(4)?; - let table = schema::BTreeTable::from_sql(sql, root_page as usize)?; - schema.add_table(Rc::new(table)); - } - "index" => { - let root_page: i64 = row.get::(3)?; - match row.get::<&str>(4) { - Ok(sql) => { - let index = - schema::Index::from_sql(sql, root_page as usize)?; - schema.add_index(Rc::new(index)); - } - _ => continue, - // TODO parse auto index structures - } - } - _ => continue, - } - } - RowResult::IO => { - // TODO: How do we ensure that the I/O we submitted to - // read the schema is actually complete? - io.run_once()?; - } - RowResult::Done => break, - } - } - } - let schema = Rc::new(schema); + parse_schema_rows(rows, &mut schema, io)?; + let schema = Rc::new(RefCell::new(schema)); let header = db_header; Ok(Rc::new(Database { pager, @@ -208,7 +171,7 @@ pub fn maybe_init_database_file(file: &Rc, io: &Arc) -> Result pub struct Connection { pager: Rc, - schema: Rc, + schema: Rc>, header: Rc>, db: Weak, // backpointer to the database holding this connection } @@ -223,7 +186,8 @@ impl Connection { match cmd { Cmd::Stmt(stmt) => { let program = Rc::new(translate::translate( - &self.schema, + sql.as_str(), + &*self.schema.borrow(), stmt, self.header.clone(), self.pager.clone(), @@ -248,7 +212,8 @@ impl Connection { match cmd { Cmd::Stmt(stmt) => { let program = Rc::new(translate::translate( - &self.schema, + sql.as_str(), + &*self.schema.borrow(), stmt, self.header.clone(), self.pager.clone(), @@ -259,7 +224,8 @@ impl Connection { } Cmd::Explain(stmt) => { let program = translate::translate( - &self.schema, + sql.as_str(), + &*self.schema.borrow(), stmt, self.header.clone(), self.pager.clone(), @@ -271,7 +237,7 @@ impl Connection { Cmd::ExplainQueryPlan(stmt) => { match stmt { ast::Stmt::Select(select) => { - let plan = prepare_select_plan(&self.schema, select)?; + let plan = prepare_select_plan(&*self.schema.borrow(), select)?; let (plan, _) = optimize_plan(plan)?; println!("{}", plan); } @@ -293,7 +259,8 @@ impl Connection { match cmd { Cmd::Explain(stmt) => { let program = translate::translate( - &self.schema, + sql.as_str(), + &*self.schema.borrow(), stmt, self.header.clone(), self.pager.clone(), @@ -304,7 +271,8 @@ impl Connection { Cmd::ExplainQueryPlan(_stmt) => todo!(), Cmd::Stmt(stmt) => { let program = translate::translate( - &self.schema, + sql.as_str(), + &*self.schema.borrow(), stmt, self.header.clone(), self.pager.clone(), diff --git a/core/pseudo.rs b/core/pseudo.rs index 73aee406..f4b76cf6 100644 --- a/core/pseudo.rs +++ b/core/pseudo.rs @@ -87,4 +87,8 @@ impl Cursor for PseudoCursor { let _ = key; todo!() } + + fn create_tree(&mut self, _flags: usize) -> u32 { + unreachable!("Please don't.") + } } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 6383d065..825e1f9d 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1676,6 +1676,20 @@ impl Cursor for BTreeCursor { Ok(CursorResult::Ok(equals)) } } + + fn create_tree(&mut self, flags: usize) -> u32 { + let page_type = match flags { + 1 => PageType::TableLeaf, + 2 => PageType::IndexLeaf, + _ => unreachable!( + "wrong create table falgs, should be 1 for table and 2 for index, got {}", + flags, + ), + }; + let page = self.allocate_page(page_type); + let id = page.borrow().id; + id as u32 + } } pub fn btree_init_page(page: &Rc>, page_type: PageType, db_header: &DatabaseHeader) { diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 17ed9dec..c78b196a 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -29,6 +29,7 @@ use sqlite3_parser::ast; /// Translate SQL statement into bytecode program. pub fn translate( + sql: &str, // raw sql query before parsing into stmt, useful for CREATE TABLE schema: &Schema, stmt: ast::Stmt, database_header: Rc>, @@ -52,6 +53,7 @@ pub fn translate( bail_parse_error!("TEMPORARY table not supported yet"); } translate_create_table( + sql, tbl_name, body, if_not_exists, @@ -142,6 +144,7 @@ addr opcode p1 p2 p3 p4 p5 comment */ fn translate_create_table( + sql: &str, tbl_name: ast::QualifiedName, body: ast::CreateTableBody, if_not_exists: bool, @@ -185,61 +188,12 @@ fn translate_create_table( rowid_reg, prev_largest_reg: 0, }); - let blob_reg = program.alloc_register(); - program.emit_insn(Insn::Blob { - value: vec![0, 0, 0, 0, 0, 0], - dest: blob_reg, - }); - program.emit_insn(Insn::InsertAsync { - cursor: sqlite_schema_cursor_id, - key_reg: rowid_reg, - record_reg: blob_reg, - flag: 0, - }); - program.emit_insn(Insn::InsertAwait { - cursor_id: sqlite_schema_cursor_id, - }); - program.emit_insn(Insn::Close { - cursor_id: sqlite_schema_cursor_id, - }); - program.emit_insn(Insn::Close { - cursor_id: sqlite_schema_cursor_id, - }); let null_reg_1 = program.alloc_register(); let null_reg_2 = program.alloc_register(); program.emit_insn(Insn::Null { dest: null_reg_1, dest_end: Some(null_reg_2), }); - // TODO: Noop - let sqlite_schema_cursor_id = program.alloc_cursor_id(Some(table_id), Some(table)); - program.emit_insn(Insn::OpenWriteAsync { - cursor_id: sqlite_schema_cursor_id, - root_page: 1, - }); - program.emit_insn(Insn::OpenWriteAwait {}); - let seek_failed_label = program.allocate_label(); - program.emit_insn_with_label_dependency( - Insn::SeekRowid { - cursor_id: sqlite_schema_cursor_id, - src_reg: rowid_reg, - target_pc: seek_failed_label, - }, - seek_failed_label, - ); - let rowid_reg_2 = program.alloc_register(); - program.emit_insn(Insn::RowId { - cursor_id: sqlite_schema_cursor_id, - dest: rowid_reg_2, - }); - program.resolve_label(seek_failed_label, program.offset()); - program.emit_insn_with_label_dependency( - Insn::IsNull { - src: rowid_reg, - target_pc: parse_schema_label, - }, - parse_schema_label, - ); let type_reg = program.alloc_register(); program.emit_insn(Insn::String8 { value: "table".to_string(), @@ -263,20 +217,35 @@ fn translate_create_table( }); let sql_reg = program.alloc_register(); program.emit_insn(Insn::String8 { - value: "TODO".to_string(), + value: sql.to_string(), dest: sql_reg, }); let record_reg = program.alloc_register(); program.emit_insn(Insn::MakeRecord { start_reg: type_reg, - count: 4, + count: 5, dest_reg: record_reg, }); // TODO: Delete - // TODO: Insert + program.emit_insn(Insn::InsertAsync { + cursor: sqlite_schema_cursor_id, + key_reg: rowid_reg, + record_reg, + flag: 0, + }); + program.emit_insn(Insn::InsertAwait { + cursor_id: sqlite_schema_cursor_id, + }); program.resolve_label(parse_schema_label, program.offset()); // TODO: SetCookie - // TODO: ParseSchema + // + // TODO: remove format, it sucks for performance but is convinient + let parse_schema_where_clause = format!("tbl_name = '{}' AND type != 'trigger'", tbl_name); + program.emit_insn(Insn::ParseSchema { + db: sqlite_schema_cursor_id, + where_clause: parse_schema_where_clause, + }); + // TODO: SqlExec program.emit_insn(Insn::Halt { err_code: 0, diff --git a/core/types.rs b/core/types.rs index 9b24b2fb..42ee2bc4 100644 --- a/core/types.rs +++ b/core/types.rs @@ -373,6 +373,7 @@ impl OwnedRecord { let mut header_size = buf.len() - initial_i; // write content for value in &self.values { + // TODO: make integers and floats with smaller serial types match value { OwnedValue::Null => {} OwnedValue::Integer(i) => buf.extend_from_slice(&i.to_be_bytes()), @@ -440,4 +441,5 @@ pub trait Cursor { fn exists(&mut self, key: &OwnedValue) -> Result>; fn set_null_flag(&mut self, flag: bool); fn get_null_flag(&self) -> bool; + fn create_tree(&mut self, flags: usize) -> u32; } diff --git a/core/util.rs b/core/util.rs index 32dc322c..ed1e6001 100644 --- a/core/util.rs +++ b/core/util.rs @@ -1,3 +1,10 @@ +use std::{rc::Rc, sync::Arc}; + +use crate::{ + schema::{self, Schema}, + Result, RowResult, Rows, IO, +}; + pub fn normalize_ident(ident: &str) -> String { (if ident.starts_with('"') && ident.ends_with('"') { &ident[1..ident.len() - 1] @@ -6,3 +13,45 @@ pub fn normalize_ident(ident: &str) -> String { }) .to_lowercase() } + +pub fn parse_schema_rows(rows: Option, schema: &mut Schema, io: Arc) -> Result<()> { + if let Some(mut rows) = rows { + loop { + match rows.next_row()? { + RowResult::Row(row) => { + let ty = row.get::<&str>(0)?; + if ty != "table" && ty != "index" { + continue; + } + match ty { + "table" => { + let root_page: i64 = row.get::(3)?; + let sql: &str = row.get::<&str>(4)?; + let table = schema::BTreeTable::from_sql(sql, root_page as usize)?; + schema.add_table(Rc::new(table)); + } + "index" => { + let root_page: i64 = row.get::(3)?; + match row.get::<&str>(4) { + Ok(sql) => { + let index = schema::Index::from_sql(sql, root_page as usize)?; + schema.add_index(Rc::new(index)); + } + _ => continue, + // TODO parse auto index structures + } + } + _ => continue, + } + } + RowResult::IO => { + // TODO: How do we ensure that the I/O we submitted to + // read the schema is actually complete? + io.run_once()?; + } + RowResult::Done => break, + } + } + } + Ok(()) +} diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index ae6f4ec7..7232c380 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -893,6 +893,15 @@ pub fn insn_to_str( 0, format!("if (r[{}]==NULL) goto {}", src, target_pc), ), + Insn::ParseSchema { db, where_clause } => ( + "ParseSchema", + *db as i32, + 0, + 0, + OwnedValue::Text(Rc::new(where_clause.clone())), + 0, + where_clause.clone(), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 3d15c549..1e2f5715 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -32,10 +32,11 @@ use crate::storage::{btree::BTreeCursor, pager::Pager}; use crate::types::{ AggContext, Cursor, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp, }; -use crate::DATABASE_VERSION; +use crate::util::parse_schema_rows; #[cfg(feature = "json")] use crate::{function::JsonFunc, json::get_json}; use crate::{Connection, Result, TransactionState}; +use crate::{Rows, DATABASE_VERSION}; use datetime::{exec_date, exec_time, exec_unixepoch}; @@ -493,6 +494,10 @@ pub enum Insn { /// Jump to this PC if the register is null (P2). target_pc: BranchOffset, }, + ParseSchema { + db: usize, + where_clause: String, + }, } // Index of insn in list of insns @@ -2111,8 +2116,20 @@ impl Program { } state.pc += 1; } - Insn::CreateBtree { .. } => { - todo!(); + Insn::CreateBtree { db, root, flags: _ } => { + if *db > 0 { + // TODO: implement temp datbases + todo!("temp databases not implemented yet"); + } + let mut cursor = Box::new(BTreeCursor::new( + pager.clone(), + 0, + self.database_header.clone(), + )); + + let root_page = cursor.create_tree(1); + state.registers[*root] = OwnedValue::Integer(root_page as i64); + state.pc += 1; } Insn::Close { cursor_id } => { cursors.remove(cursor_id); @@ -2125,12 +2142,29 @@ impl Program { state.pc += 1; } } + Insn::ParseSchema { + db: _, + where_clause, + } => { + let conn = self.connection.upgrade(); + let conn = conn.as_ref().unwrap(); + let stmt = conn.prepare(format!( + "SELECT * FROM sqlite_schema WHERE {}", + where_clause + ))?; + let rows = Rows { stmt }; + let mut schema = RefCell::borrow_mut(&conn.schema); + // TODO: This function below is synchronous, make it not async + parse_schema_rows(Some(rows), &mut *schema, conn.pager.io.clone())?; + state.pc += 1; + } } } } } fn get_new_rowid(cursor: &mut Box, mut rng: R) -> Result> { + // TODO: try last + 1 and if it's already a big number then use randomness cursor.seek_to_last()?; let mut rowid = cursor.rowid()?.unwrap_or(0) + 1; if rowid > i64::MAX.try_into().unwrap() { diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index 75afdef3..c22b3c22 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -87,4 +87,8 @@ impl Cursor for Sorter { let _ = key; todo!() } + + fn create_tree(&mut self, _flags: usize) -> u32 { + unreachable!("Why did you try to build a new tree with a sorter??? Stand up, open the door and take a walk for 30 min to come back with a better plan."); + } } From d1dee484df5783dc9566ee84d256f5feff9751ba Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Sat, 16 Nov 2024 17:02:53 +0100 Subject: [PATCH 4/7] properly format table and "if not exists" support --- core/lib.rs | 5 ---- core/translate/mod.rs | 56 ++++++++++++++++++++++++++++++++++++++++--- core/vdbe/builder.rs | 2 +- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index cdb35549..21650f46 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -186,7 +186,6 @@ impl Connection { match cmd { Cmd::Stmt(stmt) => { let program = Rc::new(translate::translate( - sql.as_str(), &*self.schema.borrow(), stmt, self.header.clone(), @@ -212,7 +211,6 @@ impl Connection { match cmd { Cmd::Stmt(stmt) => { let program = Rc::new(translate::translate( - sql.as_str(), &*self.schema.borrow(), stmt, self.header.clone(), @@ -224,7 +222,6 @@ impl Connection { } Cmd::Explain(stmt) => { let program = translate::translate( - sql.as_str(), &*self.schema.borrow(), stmt, self.header.clone(), @@ -259,7 +256,6 @@ impl Connection { match cmd { Cmd::Explain(stmt) => { let program = translate::translate( - sql.as_str(), &*self.schema.borrow(), stmt, self.header.clone(), @@ -271,7 +267,6 @@ impl Connection { Cmd::ExplainQueryPlan(_stmt) => todo!(), Cmd::Stmt(stmt) => { let program = translate::translate( - sql.as_str(), &*self.schema.borrow(), stmt, self.header.clone(), diff --git a/core/translate/mod.rs b/core/translate/mod.rs index c78b196a..cb246323 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod planner; pub(crate) mod select; use std::cell::RefCell; +use std::fmt::Display; use std::rc::{Rc, Weak}; use crate::schema::Schema; @@ -26,10 +27,10 @@ use crate::{bail_parse_error, Connection, Result}; use insert::translate_insert; use select::translate_select; use sqlite3_parser::ast; +use sqlite3_parser::ast::fmt::ToTokens; /// Translate SQL statement into bytecode program. pub fn translate( - sql: &str, // raw sql query before parsing into stmt, useful for CREATE TABLE schema: &Schema, stmt: ast::Stmt, database_header: Rc>, @@ -53,7 +54,6 @@ pub fn translate( bail_parse_error!("TEMPORARY table not supported yet"); } translate_create_table( - sql, tbl_name, body, if_not_exists, @@ -144,7 +144,6 @@ addr opcode p1 p2 p3 p4 p5 comment */ fn translate_create_table( - sql: &str, tbl_name: ast::QualifiedName, body: ast::CreateTableBody, if_not_exists: bool, @@ -153,6 +152,33 @@ fn translate_create_table( schema: &Schema, ) -> Result { let mut program = ProgramBuilder::new(); + if schema.get_table(tbl_name.name.0.as_str()).is_some() { + if if_not_exists { + let init_label = program.allocate_label(); + program.emit_insn_with_label_dependency( + Insn::Init { + target_pc: init_label, + }, + init_label, + ); + let start_offset = program.offset(); + program.emit_insn(Insn::Halt { + err_code: 0, + description: String::new(), + }); + program.resolve_label(init_label, program.offset()); + program.emit_insn(Insn::Transaction { write: true }); + program.emit_constant_insns(); + program.emit_insn(Insn::Goto { + target_pc: start_offset, + }); + return Ok(program.build(database_header, connection)); + } + bail_parse_error!("Table {} already exists", tbl_name); + } + + let sql = create_table_body_to_str(&tbl_name, &body); + let parse_schema_label = program.allocate_label(); let init_label = program.allocate_label(); program.emit_insn_with_label_dependency( @@ -362,3 +388,27 @@ fn update_pragma(name: &str, value: i64, header: Rc>, pa _ => todo!(), } } + +struct TableFormatter<'a> { + body: &'a ast::CreateTableBody, +} +impl<'a> Display for TableFormatter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.body.to_fmt(f) + } +} + +fn create_table_body_to_str(tbl_name: &ast::QualifiedName, body: &ast::CreateTableBody) -> String { + let mut sql = String::new(); + let formatter = TableFormatter { body }; + sql.push_str(format!("CREATE TABLE {} {}", tbl_name.name.0, formatter).as_str()); + match body { + ast::CreateTableBody::ColumnsAndConstraints { + columns: _, + constraints: _, + options: _, + } => {} + ast::CreateTableBody::AsSelect(_select) => todo!("as select not yet supported"), + } + sql +} diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index e9d8ac95..2c7561c2 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -319,7 +319,7 @@ impl ProgramBuilder { assert!(*target_pc < 0); *target_pc = to_offset; } - Insn::IsNull { src, target_pc } => { + Insn::IsNull { src: _, target_pc } => { assert!(*target_pc < 0); *target_pc = to_offset; } From 569e98cb32467e0bec950ff687305e7de662cfbd Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Sat, 16 Nov 2024 17:08:22 +0100 Subject: [PATCH 5/7] fmt --- core/vdbe/mod.rs | 4 ++++ sqlite3/src/lib.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 1e2f5715..adfb93a2 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -2825,6 +2825,10 @@ mod tests { fn exists(&mut self, _key: &OwnedValue) -> Result> { unimplemented!() } + + fn create_tree(&mut self, _flags: usize) -> u32 { + unimplemented!() + } } #[test] diff --git a/sqlite3/src/lib.rs b/sqlite3/src/lib.rs index ac37d8ea..67c975fb 100644 --- a/sqlite3/src/lib.rs +++ b/sqlite3/src/lib.rs @@ -885,7 +885,7 @@ fn sqlite3_errstr_impl(rc: i32) -> *const std::ffi::c_char { "datatype mismatch", // SQLITE_MISMATCH "bad parameter or other API misuse", // SQLITE_MISUSE #[cfg(feature = "lfs")] - "", // SQLITE_NOLFS + "", // SQLITE_NOLFS #[cfg(not(feature = "lfs"))] "large file support is disabled", // SQLITE_NOLFS "authorization denied", // SQLITE_AUTH From 9f72655e30c032f1cc54cd40ca75e4eb49dba987 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Mon, 18 Nov 2024 10:31:47 +0100 Subject: [PATCH 6/7] tree_create -> btree_create --- core/pseudo.rs | 2 +- core/storage/btree.rs | 2 +- core/types.rs | 2 +- core/vdbe/mod.rs | 4 ++-- core/vdbe/sorter.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/pseudo.rs b/core/pseudo.rs index f4b76cf6..5c1e445a 100644 --- a/core/pseudo.rs +++ b/core/pseudo.rs @@ -88,7 +88,7 @@ impl Cursor for PseudoCursor { todo!() } - fn create_tree(&mut self, _flags: usize) -> u32 { + fn btree_create(&mut self, _flags: usize) -> u32 { unreachable!("Please don't.") } } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 825e1f9d..3a60937e 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1677,7 +1677,7 @@ impl Cursor for BTreeCursor { } } - fn create_tree(&mut self, flags: usize) -> u32 { + fn btree_create(&mut self, flags: usize) -> u32 { let page_type = match flags { 1 => PageType::TableLeaf, 2 => PageType::IndexLeaf, diff --git a/core/types.rs b/core/types.rs index 42ee2bc4..cbe98993 100644 --- a/core/types.rs +++ b/core/types.rs @@ -441,5 +441,5 @@ pub trait Cursor { fn exists(&mut self, key: &OwnedValue) -> Result>; fn set_null_flag(&mut self, flag: bool); fn get_null_flag(&self) -> bool; - fn create_tree(&mut self, flags: usize) -> u32; + fn btree_create(&mut self, flags: usize) -> u32; } diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index adfb93a2..596bbc10 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -2127,7 +2127,7 @@ impl Program { self.database_header.clone(), )); - let root_page = cursor.create_tree(1); + let root_page = cursor.btree_create(1); state.registers[*root] = OwnedValue::Integer(root_page as i64); state.pc += 1; } @@ -2826,7 +2826,7 @@ mod tests { unimplemented!() } - fn create_tree(&mut self, _flags: usize) -> u32 { + fn btree_create(&mut self, _flags: usize) -> u32 { unimplemented!() } } diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index c22b3c22..f7eb4c43 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -88,7 +88,7 @@ impl Cursor for Sorter { todo!() } - fn create_tree(&mut self, _flags: usize) -> u32 { + fn btree_create(&mut self, _flags: usize) -> u32 { unreachable!("Why did you try to build a new tree with a sorter??? Stand up, open the door and take a walk for 30 min to come back with a better plan."); } } From 80f60852df40358205bb9c141b27251881b9243c Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Mon, 18 Nov 2024 10:41:20 +0100 Subject: [PATCH 7/7] remove wrong comment --- core/vdbe/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 596bbc10..7f91d40a 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -2164,7 +2164,6 @@ impl Program { } fn get_new_rowid(cursor: &mut Box, mut rng: R) -> Result> { - // TODO: try last + 1 and if it's already a big number then use randomness cursor.seek_to_last()?; let mut rowid = cursor.rowid()?.unwrap_or(0) + 1; if rowid > i64::MAX.try_into().unwrap() {