Merge 'Macro for generating opcode description from Rustdoc' from Vig…

…nesh

Reference Issue: #393
This PR implements a procedural macro derive_description that automates
the generation of a get_description method for enums. The macro extracts
documentation comments (specified with `/// Description...`) associated
with enum variants and generates an implementation that provides
optional descriptions for each variant.

Closes #528

Cargo.toml

@@ -9,7 +9,7 @@ members = [
     "sqlite3",
     "core",
     "simulator",
-    "test",
+    "test", "macros",
 ]
 exclude = ["perf/latency/limbo"]
 

core/Cargo.toml

@@ -53,6 +53,7 @@ pest = { version = "2.0", optional = true }
 pest_derive = { version = "2.0", optional = true }
 rand = "0.8.5"
 bumpalo = { version = "3.16.0", features = ["collections", "boxed"] }
+macros = { path = "../macros" }
 
 [target.'cfg(not(target_family = "windows"))'.dev-dependencies]
 pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] }

core/vdbe/mod.rs

@@ -48,9 +48,8 @@ use std::cell::RefCell;
 use std::collections::{BTreeMap, HashMap};
 use std::fmt::Display;
 use std::rc::{Rc, Weak};
-
 pub type BranchOffset = i64;
-
+use macros::Description;
 pub type CursorID = usize;
 
 pub type PageIdx = usize;
@@ -74,7 +73,7 @@ impl Display for Func {
     }
 }
 
-#[derive(Debug)]
+#[derive(Description, Debug)]
 pub enum Insn {
     // Initialize the program state and jump to the given PC.
     Init {

macros/Cargo.toml

@@ -0,0 +1,5 @@
+[package]
+name = "macros"
+
+[lib]
+proc-macro = true

macros/src/lib.rs

@@ -0,0 +1,137 @@
+extern crate proc_macro;
+use proc_macro::{token_stream::IntoIter, Group, TokenStream, TokenTree};
+use std::collections::HashMap;
+
+/// A procedural macro that derives a `Description` trait for enums.
+/// This macro extracts documentation comments (specified with `/// Description...`) for enum variants
+/// and generates an implementation for `get_description`, which returns the associated description.
+#[proc_macro_derive(Description, attributes(desc))]
+pub fn derive_description_from_doc(item: TokenStream) -> TokenStream {
+    // Convert the TokenStream into an iterator of TokenTree
+    let mut tokens = item.into_iter();
+
+    let mut enum_name = String::new();
+
+    // Vector to store enum variants and their associated payloads (if any)
+    let mut enum_variants: Vec<(String, Option<String>)> = Vec::<(String, Option<String>)>::new();
+
+    // HashMap to store descriptions associated with each enum variant
+    let mut variant_description_map: HashMap<String, String> = HashMap::new();
+
+    // Parses the token stream to extract the enum name and its variants
+    while let Some(token) = tokens.next() {
+        match token {
+            TokenTree::Ident(ident) if ident.to_string() == "enum" => {
+                // Get the enum name
+                if let Some(TokenTree::Ident(name)) = tokens.next() {
+                    enum_name = name.to_string();
+                }
+            }
+            TokenTree::Group(group) => {
+                let mut group_tokens_iter: IntoIter = group.stream().into_iter();
+
+                let mut last_seen_desc: Option<String> = None;
+                while let Some(token) = group_tokens_iter.next() {
+                    match token {
+                        TokenTree::Punct(punct) => {
+                            if punct.to_string() == "#" {
+                                last_seen_desc = process_description(&mut group_tokens_iter);
+                            }
+                        }
+                        TokenTree::Ident(ident) => {
+                            // Capture the enum variant name and associate it with its description
+                            let ident_str = ident.to_string();
+                            if let Some(desc) = &last_seen_desc {
+                                variant_description_map.insert(ident_str.clone(), desc.clone());
+                            }
+                            enum_variants.push((ident_str, None));
+                            last_seen_desc = None;
+                        }
+                        TokenTree::Group(group) => {
+                            // Capture payload information for the current enum variant
+                            if let Some(last_variant) = enum_variants.last_mut() {
+                                last_variant.1 = Some(process_payload(group));
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+    generate_get_description(enum_name, &variant_description_map, enum_variants)
+}
+
+/// Processes a Rust docs to extract the description string.
+fn process_description(token_iter: &mut IntoIter) -> Option<String> {
+    if let Some(doc_token_tree) = token_iter.next() {
+        if let TokenTree::Group(doc_group) = doc_token_tree {
+            let mut doc_group_iter = doc_group.stream().into_iter();
+            // Skip the `desc` and `(` tokens to reach the actual description
+            doc_group_iter.next();
+            doc_group_iter.next();
+            if let Some(TokenTree::Literal(description)) = doc_group_iter.next() {
+                return Some(description.to_string());
+            }
+        }
+    }
+    None
+}
+
+/// Processes the payload of an enum variant to extract variable names (ignoring types).
+fn process_payload(payload_group: Group) -> String {
+    let mut payload_group_iter = payload_group.stream().into_iter();
+    let mut variable_name_list = String::from("");
+    let mut is_variable_name = true;
+    while let Some(token) = payload_group_iter.next() {
+        match token {
+            TokenTree::Ident(ident) => {
+                if is_variable_name {
+                    variable_name_list.push_str(&format!("{},", ident.to_string()));
+                }
+                is_variable_name = false;
+            }
+            TokenTree::Punct(punct) => {
+                if punct.to_string() == "," {
+                    is_variable_name = true;
+                }
+            }
+            _ => {}
+        }
+    }
+    format!("{{ {} }}", variable_name_list).to_string()
+}
+/// Generates the `get_description` implementation for the processed enum.
+fn generate_get_description(
+    enum_name: String,
+    variant_description_map: &HashMap<String, String>,
+    enum_variants: Vec<(String, Option<String>)>,
+) -> TokenStream {
+    let mut all_enum_arms = String::from("");
+    for (variant, payload) in enum_variants {
+        let payload = payload.unwrap_or("".to_string());
+        let desc;
+        if let Some(description) = variant_description_map.get(&variant) {
+            desc = format!("Some({})", description);
+        } else {
+            desc = "None".to_string();
+        }
+        all_enum_arms.push_str(&format!(
+            "{}::{} {} => {},\n",
+            enum_name, variant, payload, desc
+        ));
+    }
+
+    let enum_impl = format!(
+        "impl {}  {{ 
+     pub fn get_description(&self) -> Option<&str> {{
+     match self {{
+     {}
+     }}
+     }}
+     }}",
+        enum_name, all_enum_arms
+    );
+    enum_impl.parse().unwrap()
+}