add tests

Cargo.toml

@@ -10,3 +10,7 @@ logos = "0.13.0"
 rowan-test = { git = "https://github.com/avast/avast-rowan.git" }
 text-size = "1.1.1"
 drop_bomb = "0.1.5"
+
+[dev-dependencies]
+goldenfile = "1.6.0"
+globwalk = "0.9.1"

src/main.rs

@@ -1,7 +1,7 @@
 /// This library is used to create a parser for YARA language
 /// It should provide also token for whitespaces
 /// as we want full fidelity and error resilience.;
-use std::{env::args, fs, path::Path};
+use std::{env::args, fs, io::Write, path::Path};
 
 use rowan_test::{GreenNode, NodeOrToken};
 
@@ -42,27 +42,68 @@ fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     println!();
 
     let indent = 0;
-    print(indent, syntax_tree.into());
-    //for child in syntax_tree.children() {
-    //    print!("{:indent$}", "", indent = indent);
-    //    println!("{:?}", child.kind());
-    //    println!("{:?}", child.green().children());
-    //}
+    let result = print(indent, syntax_tree.into());
+
+    print!("{}", result);
 
     (tree, parser_errors)
 }
 
-fn print(indent: usize, element: SyntaxElement) {
+fn print(indent: usize, element: SyntaxElement) -> String {
+    let mut result = String::new();
     let kind: SyntaxKind = element.kind();
-    print!("{:indent$}", "", indent = indent);
+    result.push_str(&format!("{:indent$}", "", indent = indent));
     match element {
         NodeOrToken::Node(node) => {
-            println!("- {:?}", kind);
+            result.push_str(&format!("- {:?}\n", kind));
             for child in node.children_with_tokens() {
-                print(indent + 2, child);
+                result.push_str(&print(indent + 2, child));
             }
         }
 
-        NodeOrToken::Token(token) => println!("- {:?} {:?}", token.text(), kind),
+        NodeOrToken::Token(token) => {
+            result.push_str(&format!("- {:?} {:?}\n", token.text(), kind));
+        }
+    }
+    result
+}
+
+#[test]
+fn test_parse_text() {
+    let mut mint = goldenfile::Mint::new(".");
+
+    for entry in globwalk::glob("tests/*.in").unwrap().flatten() {
+        // Path to the .in.zip file.
+        let path = entry.into_path();
+        let display_path = path.display();
+
+        let input = fs::read_to_string(&path)
+            .unwrap_or_else(|_| panic!("Failed to read input file {:?}", display_path));
+
+        let (tree, errors) = parse_text(&input);
+
+        let out_path = path.with_extension("").with_extension("out");
+        let syntax_tree = SyntaxNode::new_root(tree.clone());
+
+        let output = print(0, syntax_tree.into());
+
+        let mut output_file = mint.new_goldenfile(out_path).unwrap();
+
+        write!(output_file, "{}", output).unwrap();
+
+        // Check errors
+        let err_path = path.with_extension("").with_extension("err");
+        if err_path.exists() {
+            let expected_errors = fs::read_to_string(&err_path)
+                .unwrap_or_else(|_| panic!("Failed to read error file {:?}", err_path.display()));
+            let actual_errors = errors
+                .iter()
+                .map(|error| format!("{:?}", error))
+                .collect::<Vec<_>>()
+                .join("\n");
+            assert_eq!(actual_errors, expected_errors);
+        } else {
+            assert!(errors.is_empty(), "Unexpected errors: {:?}", errors);
+        }
     }
 }

src/parser/grammar/expressions.rs

@@ -45,7 +45,8 @@ fn condition(p: &mut Parser) {
     m.complete(p, CONDITION);
 }
 
-const VARIABLE_RECOVERY_SET: TokenSet = TokenSet::new(&[VARIABLE]);
+const VARIABLE_RECOVERY_SET: TokenSet =
+    TokenSet::new(&[VARIABLE, CONDITION, STRINGS, ASSIGN, RBRACE]);
 
 pub(super) fn strings_body(p: &mut Parser) {
     // add support for meta also

src/parser/parser.rs

@@ -111,15 +111,8 @@ impl<'t> Parser<'t> {
     }
 
     pub(crate) fn err_recover(&mut self, message: &str, recovery: TokenSet) {
-        match self.current() {
-            LBRACE | RBRACE => {
-                self.error(message);
-                return;
-            }
-            _ => (),
-        }
-
         if self.at_ts(recovery) {
+            println!("recovery: {:?}", self.current());
             self.error(message);
             return;
         }

tests/test1.in

@@ -0,0 +1,7 @@
+rule test
+{
+	strings:
+		$a = "foo"
+	condition:
+		$a
+}

tests/test1.out

@@ -0,0 +1,33 @@
+- SOURCE_FILE
+  - RULE
+    - "rule" RULE
+    - " " WHITESPACE
+    - IDENTIFIER
+      - "test" IDENTIFIER
+    - "\n" WHITESPACE
+    - BLOCK_EXPR
+      - "{" LBRACE
+      - "\n\t" WHITESPACE
+      - STRINGS
+        - "strings" STRINGS
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$a" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"foo\"" STRING
+      - "\n\t" WHITESPACE
+      - CONDITION
+        - "condition" CONDITION
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - EXPRESSION_STMT
+          - LITERAL
+            - "$a" VARIABLE
+      - "\n" WHITESPACE
+      - "}" RBRACE
+  - "\n" WHITESPACE

tests/test2.in

@@ -0,0 +1,9 @@
+rule test
+{
+	strings:
+		$a = "foo"
+		$b = "bar"
+	condition:
+		$a or
+		$b
+}

tests/test2.out

@@ -0,0 +1,48 @@
+- SOURCE_FILE
+  - RULE
+    - "rule" RULE
+    - " " WHITESPACE
+    - IDENTIFIER
+      - "test" IDENTIFIER
+    - "\n" WHITESPACE
+    - BLOCK_EXPR
+      - "{" LBRACE
+      - "\n\t" WHITESPACE
+      - STRINGS
+        - "strings" STRINGS
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$a" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"foo\"" STRING
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$b" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"bar\"" STRING
+      - "\n\t" WHITESPACE
+      - CONDITION
+        - "condition" CONDITION
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - EXPRESSION_STMT
+          - EXPRESSION
+            - LITERAL
+              - "$a" VARIABLE
+            - " " WHITESPACE
+            - "or" OR
+            - "\n\t\t" WHITESPACE
+            - LITERAL
+              - "$b" VARIABLE
+      - "\n" WHITESPACE
+      - "}" RBRACE
+  - "\n" WHITESPACE

tests/test3.in

@@ -0,0 +1,15 @@
+//Global comment
+
+//Rule comment
+rule test
+{
+	//Rule block comment
+
+	//String comment
+	strings:
+		$a = "foo"
+		$b = "bar"
+	condition:
+		$a or
+		$b and true
+}

tests/test3.out

@@ -0,0 +1,62 @@
+- SOURCE_FILE
+  - "//Global comment" COMMENT
+  - "\n\n" WHITESPACE
+  - RULE
+    - "//Rule comment" COMMENT
+    - "\n" WHITESPACE
+    - "rule" RULE
+    - " " WHITESPACE
+    - IDENTIFIER
+      - "test" IDENTIFIER
+    - "\n" WHITESPACE
+    - BLOCK_EXPR
+      - "{" LBRACE
+      - "\n\t" WHITESPACE
+      - "//Rule block comment" COMMENT
+      - "\n\n\t" WHITESPACE
+      - STRINGS
+        - "//String comment" COMMENT
+        - "\n\t" WHITESPACE
+        - "strings" STRINGS
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$a" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"foo\"" STRING
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$b" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"bar\"" STRING
+      - "\n\t" WHITESPACE
+      - CONDITION
+        - "condition" CONDITION
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - EXPRESSION_STMT
+          - EXPRESSION
+            - LITERAL
+              - "$a" VARIABLE
+            - " " WHITESPACE
+            - "or" OR
+            - "\n\t\t" WHITESPACE
+            - EXPRESSION
+              - LITERAL
+                - "$b" VARIABLE
+              - " " WHITESPACE
+              - "and" AND
+              - " " WHITESPACE
+              - LITERAL
+                - "true" TRUE
+      - "\n" WHITESPACE
+      - "}" RBRACE
+  - "\n" WHITESPACE

tests/test4.err

@@ -0,0 +1 @@
+SyntaxError("expected a variable", 98..98)

tests/test4.in

@@ -0,0 +1,15 @@
+//Global comment
+
+//Rule comment
+rule test
+{
+	//Rule block comment
+
+	//String comment
+	strings:
+		a = "foo"
+		$b = "bar"
+	condition:
+		$a or
+		$b and true
+}

tests/test4.out

@@ -0,0 +1,62 @@
+- SOURCE_FILE
+  - "//Global comment" COMMENT
+  - "\n\n" WHITESPACE
+  - RULE
+    - "//Rule comment" COMMENT
+    - "\n" WHITESPACE
+    - "rule" RULE
+    - " " WHITESPACE
+    - IDENTIFIER
+      - "test" IDENTIFIER
+    - "\n" WHITESPACE
+    - BLOCK_EXPR
+      - "{" LBRACE
+      - "\n\t" WHITESPACE
+      - "//Rule block comment" COMMENT
+      - "\n\n\t" WHITESPACE
+      - STRINGS
+        - "//String comment" COMMENT
+        - "\n\t" WHITESPACE
+        - "strings" STRINGS
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - ERROR
+            - "a" IDENTIFIER
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"foo\"" STRING
+        - "\n\t\t" WHITESPACE
+        - VARIABLE_STMT
+          - VARIABLE
+            - "$b" VARIABLE
+          - " " WHITESPACE
+          - "=" ASSIGN
+          - " " WHITESPACE
+          - STRING
+            - "\"bar\"" STRING
+      - "\n\t" WHITESPACE
+      - CONDITION
+        - "condition" CONDITION
+        - ":" COLON
+        - "\n\t\t" WHITESPACE
+        - EXPRESSION_STMT
+          - EXPRESSION
+            - LITERAL
+              - "$a" VARIABLE
+            - " " WHITESPACE
+            - "or" OR
+            - "\n\t\t" WHITESPACE
+            - EXPRESSION
+              - LITERAL
+                - "$b" VARIABLE
+              - " " WHITESPACE
+              - "and" AND
+              - " " WHITESPACE
+              - LITERAL
+                - "true" TRUE
+      - "\n" WHITESPACE
+      - "}" RBRACE
+  - "\n" WHITESPACE

tests/test5.err

@@ -0,0 +1 @@
+SyntaxError("expected expression", 144..144)