Fix interaction of luau and lua53 for TokenReference::symbol('//=') (…

…#327) We incorrectly parse as just double slash, since we take the lua53 branch first
Kampfkarren · Nov 17, 2024 · 0361d2c · 0361d2c
1 parent b996441
commit 0361d2c
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 24 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Fixed `TokenReference::symbol("//=")` returning DoubleSlash + UnexpectedToken when `luau` and `lua53` are enabled together
+
 ## [1.1.1] - 2024-11-16
 
 ### Fixed

diff --git a/full-moon/src/tokenizer/lexer.rs b/full-moon/src/tokenizer/lexer.rs
@@ -577,32 +577,31 @@ impl Lexer {
 
             '/' => {
                 version_switch!(self.lua_version, {
-                    lua53 => {
-                        if self.source.current() == Some('/') {
-                            self.source.next();
-                            return self.create(
-                                start_position,
-                                TokenType::Symbol {
-                                    symbol: Symbol::DoubleSlash,
-                                },
-                            );
-                        }
-                    }
-                    luau => {
+                    lua53 | luau => {
                         if self.source.consume('/') {
-                            if self.source.consume('=') {
-                                return self.create(start_position, TokenType::Symbol { symbol: Symbol::DoubleSlashEqual })
-                            } else {
-                                return self.create(start_position, TokenType::Symbol { symbol: Symbol::DoubleSlash} )
-                            }
-                        } else if self.source.consume('=') {
-                            return self.create(
-                                start_position,
-                                TokenType::Symbol {
-                                    symbol: Symbol::SlashEqual,
-                                },
-                            );
+                            version_switch!(self.lua_version, {
+                                luau => {
+                                    if self.source.consume('=') {
+                                        return self.create(start_position, TokenType::Symbol { symbol: Symbol::DoubleSlashEqual })
+                                    }
+                                }
+                            });
+
+                            return self.create(start_position, TokenType::Symbol { symbol: Symbol::DoubleSlash} );
                         }
+
+                        version_switch!(self.lua_version, {
+                            luau => {
+                                if self.source.consume('=') {
+                                    return self.create(
+                                        start_position,
+                                        TokenType::Symbol {
+                                            symbol: Symbol::SlashEqual,
+                                        },
+                                    );
+                                }
+                            }
+                        });
                     }
                 });
 

diff --git a/full-moon/src/tokenizer/structs.rs b/full-moon/src/tokenizer/structs.rs
@@ -1195,3 +1195,20 @@ mod tests {
     }
 }
 */
+
+mod tests {
+    #[cfg(all(feature = "luau", feature = "lua53"))]
+    #[test]
+    fn test_token_symbol_create_double_slash_equal() {
+        use crate::tokenizer::{Symbol, TokenType};
+
+        use super::TokenReference;
+
+        assert!(matches!(
+            TokenReference::symbol("//=").unwrap().token().token_type(),
+            TokenType::Symbol {
+                symbol: Symbol::DoubleSlashEqual
+            }
+        ))
+    }
+}