Getting rid of semicolon syntax

Neon should does not need semicolons to end an expression.
I implicitly want to end an expression when a line ends, or the grouping gets closed with a right-paren.

src/compiler/mod.rs

@@ -21,6 +21,7 @@ struct Scanner {
     current: usize,
     line: u32,
     pos: u32,
+    previous_token_type: TokenType,
 }
 
 #[derive(Debug)]

src/compiler/parser/parsing.rs

@@ -71,19 +71,26 @@ impl Parser {
     }
 
     #[cfg_attr(feature = "disassemble", instrument(skip(self)))]
-    fn expression(&mut self) {
-        self.parse_precedence(Precedence::Assignment);
+    fn expression(&mut self, skip_new_lines: bool) {
+        self.parse_precedence(Precedence::Assignment, skip_new_lines);
     }
 
     #[cfg_attr(feature = "disassemble", instrument(skip(self, precedence)))]
-    fn parse_precedence(&mut self, precedence: Precedence) {
+    fn parse_precedence(&mut self, precedence: Precedence, skip_new_lines: bool) {
+        if skip_new_lines {
+            while self.check(TokenType::NewLine) {
+                self.advance();
+            }
+        }
+
         self.advance();
-        let prefix_rule = self.get_rule(self.previous_token.token_type.clone()).prefix;
-        if prefix_rule.is_none() {
+        let rule = self.get_rule(self.previous_token.token_type.clone());
+
+        if let None = rule.prefix {
             self.report_error_at_current("Expect expression");
             return;
         }
-        prefix_rule.unwrap()(self);
+        rule.prefix.unwrap()(self);
 
         while precedence as u8
             <= self
@@ -94,6 +101,12 @@ impl Parser {
             let infix_rule = self.get_rule(self.previous_token.token_type.clone()).infix;
             infix_rule.unwrap()(self);
         }
+
+        if skip_new_lines {
+            while self.check(TokenType::NewLine) {
+                self.advance();
+            }
+        }
     }
 
     #[cfg_attr(feature = "disassemble", instrument(skip(self)))]
@@ -111,15 +124,15 @@ impl Parser {
 
     #[cfg_attr(feature = "disassemble", instrument(skip(self)))]
     pub(super) fn grouping(&mut self) {
-        self.expression();
+        self.expression(true);
         self.consume(TokenType::RightParen, "Expect end of expression");
     }
 
     #[cfg_attr(feature = "disassemble", instrument(skip(self)))]
     pub(super) fn binary(&mut self) {
         let operator_type = self.previous_token.token_type.clone();
         let rule = self.get_rule(operator_type.clone());
-        self.parse_precedence(Precedence::from_u8(rule.precedence as u8 + 1));
+        self.parse_precedence(Precedence::from_u8(rule.precedence as u8 + 1), false);
 
         match operator_type {
             token_type if token_type == TokenType::BangEqual => {
@@ -157,7 +170,7 @@ impl Parser {
         let operator_type = self.previous_token.token_type.clone();
 
         // Compile the operand.
-        self.parse_precedence(Precedence::Unary);
+        self.parse_precedence(Precedence::Unary, false);
 
         // Emit the operator instruction.
         match operator_type {
@@ -176,6 +189,23 @@ impl Parser {
         self.report_error_at_current(message);
     }
 
+    pub(in crate::compiler) fn consume_either(
+        &mut self,
+        token_type_1: TokenType,
+        token_type_2: TokenType,
+        message: &str,
+    ) {
+        if self.current_token.token_type == token_type_1 {
+            self.advance();
+            return;
+        } else if self.current_token.token_type == token_type_2 {
+            self.advance();
+            return;
+        }
+
+        self.report_error_at_current(message);
+    }
+
     fn get_rule(&self, token_type: TokenType) -> &ParseRule {
         #[cfg(feature = "disassemble")]
         return self.get_rule_safe(token_type);
@@ -193,14 +223,22 @@ impl Parser {
     }
 
     fn print_statement(&mut self) {
-        self.expression();
-        self.consume(TokenType::Semicolon, "Expecting ';' at end of statement.");
+        self.expression(false);
+        self.consume_either(
+            TokenType::NewLine,
+            TokenType::Eof,
+            "Expecting '\\n' or '\\0' at end of statement.",
+        );
         self.emit_op_code(OpCode::Print);
     }
 
     fn expression_statement(&mut self) {
-        self.expression();
-        self.consume(TokenType::Semicolon, "Expecting ';' at end of expression.");
+        self.expression(false);
+        self.consume_either(
+            TokenType::NewLine,
+            TokenType::Eof,
+            "Expecting '\\n' or '\\0' at end of expression.",
+        );
         self.emit_op_code(OpCode::Pop);
     }
 

src/compiler/parser/rules.rs

@@ -56,6 +56,7 @@ lazy_static! {
         (TokenType::Minus, ParseRule::new(Some(Parser::unary), Some(Parser::binary), Precedence::Term)),
         (TokenType::Plus, ParseRule::new(None, Some(Parser::binary), Precedence::Term)),
         (TokenType::Semicolon, ParseRule::new(None, None, Precedence::None)),
+        (TokenType::NewLine, ParseRule::new(None, None, Precedence::None)),
         (TokenType::Slash, ParseRule::new(None, Some(Parser::binary), Precedence::Factor)),
         (TokenType::Star, ParseRule::new(None, Some(Parser::binary), Precedence::Factor)),
         (TokenType::Bang, ParseRule::new(Some(Parser::unary), None, Precedence::None)),

src/compiler/scanner.rs

@@ -9,18 +9,26 @@ impl Scanner {
             current: 0,
             line: 0,
             pos: 0,
+            previous_token_type: TokenType::NewLine,
         }
     }
 
     //noinspection DuplicatedCode
     pub(in crate::compiler) fn scan_token(&mut self) -> Token {
-        self.skip_whitespace();
-        self.start = self.current;
+        let mut c;
+        loop {
+            self.skip_whitespace();
+            self.start = self.current;
 
-        if self.is_at_end() {
-            return self.make_eof_token();
+            if self.is_at_end() {
+                return self.make_eof_token();
+            }
+            c = self.advance();
+            if !(self.previous_token_type == TokenType::NewLine && c == '\n') {
+                break;
+            }
         }
-        let c = self.advance();
+
         if Scanner::is_alpha(c) {
             return self.make_identifier();
         }
@@ -77,6 +85,11 @@ impl Scanner {
                     return self.make_token(TokenType::Slash);
                 }
             }
+            '\n' => {
+                self.line += 1;
+                self.pos = 0;
+                return self.make_token(TokenType::NewLine);
+            }
             '"' => return self.make_string(),
             _ => self.make_error_token("Unexpected character"),
         };
@@ -187,10 +200,6 @@ impl Scanner {
                     self.pos += 1;
                     self.advance();
                 }
-                '\n' => {
-                    self.line += 1;
-                    self.advance();
-                }
                 _ => {
                     break;
                 }
@@ -266,18 +275,21 @@ impl Scanner {
         };
     }
 
-    fn make_error_token(&self, message: &str) -> Token {
+    fn make_error_token(&mut self, message: &str) -> Token {
+        self.previous_token_type = TokenType::Error;
         Token::new(TokenType::Error, String::from(message), self.pos, self.line)
     }
 
     fn make_token(&mut self, token_type: TokenType) -> Token {
+        self.previous_token_type = token_type.clone();
         let token_str = String::from_iter(&self.source[self.start..self.current]);
         let token_str_len = token_str.len() as u32;
         let token = Token::new(token_type, token_str, self.pos, self.line);
         self.pos += token_str_len;
         token
     }
-    fn make_eof_token(&self) -> Token {
+    fn make_eof_token(&mut self) -> Token {
+        self.previous_token_type = TokenType::Eof;
         Token::new(TokenType::Eof, String::new(), self.pos, self.line)
     }
 }
@@ -346,6 +358,11 @@ mod tests {
                 tokens.push(token);
                 break;
             }
+            if token.token_type == TokenType::NewLine
+                && (tokens.len() == 0 || tokens[tokens.len() - 1].token_type == TokenType::NewLine)
+            {
+                continue;
+            }
             tokens.push(token);
         }
         tokens

src/compiler/token.rs

@@ -11,6 +11,7 @@ pub(in crate::compiler) enum TokenType {
     Minus,
     Plus,
     Semicolon,
+    NewLine,
     Slash,
     Star,
 

src/neon.n

@@ -1,2 +1,2 @@
-print "Hello" + " " + "World!";
-print 42 * 3.14;
+print "Hello" + " " + "World!"
+print 42 * 3.14

src/vm/virtual_machine.rs

@@ -228,7 +228,7 @@ mod tests {
     #[test]
     fn can_print_hello_world() {
         let program = r#"
-        print "Hello, World!";
+        print "Hello, World!"
         "#;
 
         let mut vm = super::VirtualMachine::new();
@@ -240,7 +240,7 @@ mod tests {
     #[test]
     fn can_print_the_answer_to_everything_times_pi() {
         let program = r#"
-        print 42 * 3.14;
+        print 42 * 3.14
         "#;
 
         let mut vm = super::VirtualMachine::new();
@@ -251,8 +251,8 @@ mod tests {
     #[test]
     fn can_run_multi_line_statements() {
         let program = r#"
-        print "Hello, World!";
-        print 42 * 3.14;
+        print "Hello, World!"
+        print 42 * 3.14
         "#;
 
         let mut vm = super::VirtualMachine::new();