Support LuaJIT

.github/workflows/test.yml

@@ -31,10 +31,14 @@ jobs:
         run: |
           cd full-moon
           cargo test --features lua54
+      - name: Test (LuaJIT feature)
+        run: |
+          cd full-moon
+          cargo test --features luajit
       - name: Test (all features)
         run: |
-          cd full-moon 
-          cargo test --features luau,lua52,lua53,lua54
+          cd full-moon
+          cargo test --features luau,lua52,lua53,lua54,luajit
       - name: Test (no default features)
         run: |
           cd full-moon

CHANGELOG.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **[BREAKING CHANGE]** `Symbol::PlusEqual` and friends are now only available when using Luau.
 - **[BREAKING CHANGE]** `Expression::Function((TokenReference, FunctionBody))` variant is now Boxed to ``Expression::Function(Box<(TokenReference, FunctionBody)>)` to reduce type sizes and reduce stack overflows
 - **[BREAKING CHANGE]** Associativity of union / intersection types are fixed: they are parsed as left associative, whilst originally parsed as right associative
+- **[BREAKING CHANGE]** LuaJIT parsing support is available as a separate feature flag `luajit`, rather than mixed with `lua52`. Added `LuaVersion::luajit()` to support this.
 - **[BREAKING CHANGE]** `Symbol::Ellipse` and other references of `ellipse` have been renamed to `Symbol::Ellipsis` / `ellipsis` 
 - Shebangs provide their trailing trivia more accurately to the rest of full-moon.
 - Attempting to display `StringLiteralQuoteType::Brackets` now returns an error rather than being marked as unreachable.

full-moon/Cargo.toml

@@ -11,8 +11,8 @@ keywords = ["lua", "parser", "lua51", "lua52", "luau"]
 edition = "2021"
 
 [package.metadata.docs.rs]
-# Build Locally: RUSTDOCFLAGS="--cfg doc_cfg" cargo +nightly doc --features luau,lua52,lua53,lua54  --no-deps --open
-features = ["luau", "lua52", "lua53", "lua54"]
+# Build Locally: RUSTDOCFLAGS="--cfg doc_cfg" cargo +nightly doc --features luau,lua52,lua53,lua54,luajit  --no-deps --open
+features = ["luau", "lua52", "lua53", "lua54", "luajit"]
 rustdoc-args = ["--cfg", "doc_cfg"]
 
 [features]
@@ -22,6 +22,7 @@ roblox = ["luau"]    # backwards compatibility
 lua52 = []
 lua53 = ["lua52"]
 lua54 = ["lua53"]
+luajit = []
 no-source-tests = []
 
 [dependencies]

full-moon/src/ast/mod.rs

@@ -34,9 +34,9 @@ use types::*;
 #[cfg(feature = "luau")]
 mod type_visitors;
 
-#[cfg(feature = "lua52")]
+#[cfg(any(feature = "lua52", feature = "luajit"))]
 pub mod lua52;
-#[cfg(feature = "lua52")]
+#[cfg(any(feature = "lua52", feature = "luajit"))]
 use lua52::*;
 
 #[cfg(feature = "lua54")]
@@ -412,12 +412,12 @@ pub enum Stmt {
     TypeDeclaration(TypeDeclaration),
 
     /// A goto statement, such as `goto label`
-    /// Only available when the "lua52" feature flag is enabled.
-    #[cfg(feature = "lua52")]
+    /// Only available when the "lua52" or "luajit" feature flag is enabled.
+    #[cfg(any(feature = "lua52", feature = "luajit"))]
     Goto(Goto),
     /// A label, such as `::label::`
-    /// Only available when the "lua52" feature flag is enabled.
-    #[cfg(feature = "lua52")]
+    /// Only available when the "lua52" or "luajit" feature flag is enabled.
+    #[cfg(any(feature = "lua52", feature = "luajit"))]
     Label(Label),
 }
 

full-moon/src/ast/parsers.rs

@@ -483,11 +483,11 @@ fn parse_stmt(state: &mut ParserState) -> ParserResult<StmtVariant> {
             })))
         }
 
-        #[cfg(feature = "lua52")]
+        #[cfg(any(feature = "lua52", feature = "luajit"))]
         TokenType::Symbol {
             symbol: Symbol::Goto,
         } => {
-            debug_assert!(state.lua_version().has_lua52());
+            debug_assert!(state.lua_version().has_lua52() || state.lua_version().has_luajit());
 
             let goto_token = state.consume().unwrap();
 
@@ -518,10 +518,10 @@ fn parse_stmt(state: &mut ParserState) -> ParserResult<StmtVariant> {
             }
         }
 
-        #[cfg(feature = "lua52")]
+        #[cfg(any(feature = "lua52", feature = "luajit"))]
         TokenType::Symbol {
             symbol: Symbol::TwoColons,
-        } if state.lua_version().has_lua52() => {
+        } if state.lua_version().has_lua52() || state.lua_version().has_luajit() => {
             let left_colons = state.consume().unwrap();
 
             let name = match state.current() {

full-moon/src/ast/versions.rs

@@ -4,6 +4,7 @@ const VERSION_LUAU: u8 = 1 << 0;
 const VERSION_LUA52: u8 = 1 << 1;
 const VERSION_LUA53: u8 = 1 << 2;
 const VERSION_LUA54: u8 = 1 << 3;
+const VERSION_LUAJIT: u8 = 1 << 4;
 
 /// Represents the Lua version(s) to parse as.
 /// Lua 5.1 is always included.
@@ -101,12 +102,31 @@ impl LuaVersion {
     pub fn has_lua54(self) -> bool {
         cfg!(feature = "lua54") && (self.bitfield & VERSION_LUA54 != 0)
     }
+
+    /// Creates a new LuaVersion with only LuaJIT.
+    #[cfg(feature = "luajit")]
+    pub fn luajit() -> Self {
+        Self {
+            bitfield: VERSION_LUAJIT,
+        }
+    }
+
+    /// Adds LuaJIT as a version to parse for.
+    #[cfg(feature = "luajit")]
+    pub fn with_luajit(self) -> Self {
+        self | Self::luajit()
+    }
+
+    /// Returns true if LuaJIT is enabled.
+    pub fn has_luajit(self) -> bool {
+        cfg!(feature = "luajit") && (self.bitfield & VERSION_LUAJIT != 0)
+    }
 }
 
 impl Default for LuaVersion {
     fn default() -> Self {
         Self {
-            bitfield: VERSION_LUAU | VERSION_LUA52 | VERSION_LUA53 | VERSION_LUA54,
+            bitfield: VERSION_LUAU | VERSION_LUA52 | VERSION_LUA53 | VERSION_LUA54 | VERSION_LUAJIT,
         }
     }
 }

full-moon/src/tokenizer/lexer.rs

@@ -332,7 +332,7 @@ impl Lexer {
                 if matches!(self.source.current(), Some('x' | 'X')) {
                     let hex_character = self.source.next().unwrap();
                     self.read_hex_number(hex_character, start_position)
-                } else if self.lua_version.has_luau()
+                } else if (self.lua_version.has_luau() || self.lua_version.has_luajit())
                     && matches!(self.source.current(), Some('b' | 'B'))
                 {
                     let binary_character = self.source.next().unwrap();
@@ -498,7 +498,7 @@ impl Lexer {
 
             ':' => {
                 version_switch!(self.lua_version, {
-                    lua52 | luau => {
+                    lua52 | luau | luajit => {
                         if self.source.consume(':') {
                             return self.create(
                                 start_position,
@@ -903,6 +903,10 @@ impl Lexer {
                 number.push(self.source.next().expect("peeked, but no next"));
             } else if matches!(next, 'e' | 'E') {
                 return self.read_exponent_part(start_position, number);
+            } else if self.lua_version.has_luajit()
+                && matches!(next, 'u' | 'U' | 'l' | 'L' | 'i' | 'I')
+            {
+                return self.read_luajit_number_suffix(start_position, number);
             } else {
                 break;
             }
@@ -1012,6 +1016,10 @@ impl Lexer {
                     return self.read_exponent_part(start_position, number);
                 }
 
+                'u' | 'U' | 'l' | 'L' | 'i' | 'I' if self.lua_version.has_luajit() => {
+                    return self.read_luajit_number_suffix(start_position, number);
+                }
+
                 _ => break,
             }
         }
@@ -1033,7 +1041,7 @@ impl Lexer {
         binary_character: char,
         start_position: Position,
     ) -> Option<LexerResult<Token>> {
-        debug_assert!(self.lua_version.has_luau());
+        debug_assert!(self.lua_version.has_luau() || self.lua_version.has_luajit());
 
         let mut number = String::from_iter(['0', binary_character]);
 
@@ -1043,6 +1051,10 @@ impl Lexer {
                     number.push(self.source.next().expect("peeked, but no next"));
                 }
 
+                'u' | 'U' | 'l' | 'L' | 'i' | 'I' if self.lua_version.has_luajit() => {
+                    return self.read_luajit_number_suffix(start_position, number);
+                }
+
                 _ => break,
             }
         }
@@ -1059,6 +1071,43 @@ impl Lexer {
         )
     }
 
+    fn read_luajit_number_suffix(
+        &mut self,
+        start_position: Position,
+        mut number: String,
+    ) -> Option<LexerResult<Token>> {
+        debug_assert!(self.lua_version.has_luajit());
+
+        while let Some(next) = self.source.current() {
+            if matches!(next, 'u' | 'U') {
+                number.push(self.source.next().expect("peeked, but no next"));
+                if !matches!(self.source.current(), Some('l' | 'L')) {
+                    return Some(self.eat_invalid_number(start_position, number));
+                }
+            } else if matches!(next, 'l' | 'L') {
+                number.push(self.source.next().expect("peeked, but no next"));
+                if matches!(self.source.current(), Some('l' | 'L')) {
+                    number.push(self.source.next().expect("peeked, but no next"));
+                    break;
+                } else {
+                    return Some(self.eat_invalid_number(start_position, number));
+                }
+            } else if matches!(next, 'i' | 'I') {
+                number.push(self.source.next().expect("peeked, but no next"));
+                break;
+            } else {
+                return Some(self.eat_invalid_number(start_position, number));
+            }
+        }
+
+        self.create(
+            start_position,
+            TokenType::Number {
+                text: ShortString::from(number),
+            },
+        )
+    }
+
     // (string, had to be recovered?)
     fn read_string(&mut self, quote: char) -> (TokenType, bool) {
         let quote_type = match quote {
@@ -1088,7 +1137,11 @@ impl Lexer {
             };
 
             match (escape, next) {
-                (true, 'z') if self.lua_version.has_lua52() || self.lua_version.has_luau() => {
+                (true, 'z')
+                    if self.lua_version.has_lua52()
+                        || self.lua_version.has_luau()
+                        || self.lua_version.has_luajit() =>
+                {
                     escape = false;
                     z_escaped = true;
                     literal.push('z');

full-moon/src/tokenizer/structs.rs

@@ -121,7 +121,7 @@ symbol! {
         Until => "until",
         While => "while",
 
-        [lua52] Goto => "goto",
+        [lua52 | luajit] Goto => "goto",
 
         [luau] PlusEqual => "+=",
         [luau] MinusEqual => "-=",
@@ -134,7 +134,7 @@ symbol! {
 
         [luau | lua53] Ampersand => "&",
         [luau] ThinArrow => "->",
-        [luau | lua52] TwoColons => "::",
+        [luau | lua52 | luajit] TwoColons => "::",
 
         Caret => "^",
         Colon => ":",

full-moon/src/visitors.rs

@@ -4,7 +4,7 @@ use crate::{
     tokenizer::{Token, TokenReference},
 };
 
-#[cfg(feature = "lua52")]
+#[cfg(any(feature = "lua52", feature = "luajit"))]
 use crate::ast::lua52::*;
 #[cfg(feature = "lua54")]
 use crate::ast::lua54::*;
@@ -298,7 +298,7 @@ create_visitor!(ast: {
     }
 
     // Lua 5.2
-    #[cfg(feature = "lua52")] {
+    #[cfg(any(feature = "lua52", feature = "luajit"))] {
         visit_goto => Goto,
         visit_label => Label,
     }

full-moon/tests/lua52_cases/pass/numbers/source.lua

@@ -6,10 +6,3 @@ local b = 0xA23p-4
 
 -- a mixture of both
 local c = 0X1.921FB54442D18P+1
-
--- LuaJIT numbers (ULL/LL ending for both decimal and hexidecimal, or imaginary)
--- This is in Lua 5.2 tests for simplicity
--- rewrite todo: add this back, but in a separate luajit pass folder
--- local d = 42LL
--- local e = 0x2aLL
--- local f = 12.5i