Parser rewrite (#274)

.github/workflows/test.yml

@@ -2,43 +2,43 @@ name: Test full-moon
 on:
   push:
     branches:
-    - main
+      - main
   pull_request:
     branches:
-    - main
+      - main
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Test (default features)
-      run: |
-        cd full-moon
-        cargo test
-    - name: Test (roblox feature)
-      run: |
-        cd full-moon
-        cargo test --features roblox
-    - name: Test (Lua 5.2 feature)
-      run: |
-        cd full-moon
-        cargo test --features lua52
-    - name: Test (Lua 5.3 feature)
-      run: |
-        cd full-moon
-        cargo test --features lua53
-    - name: Test (Lua 5.4 feature)
-      run: |
-        cd full-moon
-        cargo test --features lua54
-    - name: Test (all features)
-      run: |
-        cd full-moon 
-        cargo test --features roblox,lua52,lua53,lua54
-    - name: Test (no default features)
-      run: |
-        cd full-moon
-        cargo test --no-default-features --features serde
-    - name: Rustfmt
-      run: |
-        cargo fmt -- --check
+      - uses: actions/checkout@v3
+      - name: Test (default features)
+        run: |
+          cd full-moon
+          cargo test
+      - name: Test (Luau feature)
+        run: |
+          cd full-moon
+          cargo test --features luau
+      - name: Test (Lua 5.2 feature)
+        run: |
+          cd full-moon
+          cargo test --features lua52
+      - name: Test (Lua 5.3 feature)
+        run: |
+          cd full-moon
+          cargo test --features lua53
+      - name: Test (Lua 5.4 feature)
+        run: |
+          cd full-moon
+          cargo test --features lua54
+      - name: Test (all features)
+        run: |
+          cd full-moon 
+          cargo test --features luau,lua52,lua53,lua54
+      - name: Test (no default features)
+        run: |
+          cd full-moon
+          cargo test --no-default-features --features serde
+      - name: Rustfmt
+        run: |
+          cargo fmt -- --check

.vscode/settings.json

@@ -1,5 +1,3 @@
 {
-	"rust-analyzer.cargo.features": [
-		"roblox"
-	],
-}
+  "rust-analyzer.cargo.features": ["luau", "lua52", "lua53", "lua54"]
+}

CHANGELOG.md

@@ -6,7 +6,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 ### Added
+- **full-moon now has the ability to return multiple errors.** Added `parse_fallible`, which will return a struct containing the best possible AST, and a vector of errors. Read the documentation for guarantees on the partial AST.
+- The Lua version used to parse is no longer strictly based on features set, and can now be configured precisely using `LuaVersion`. `LuaVersion` is a bitfield that can attempt to parse multiple versions of Lua at once, or be used to pin down a specific version. `parse` will use the most completely available set possible (`LuaVersion::new()`), but `parse_fallible` accepts a `LuaVersion`.
 - Added support for parsing Luau's floor division assignment `//=`
+- Added `TokenizerErrorType::InvalidNumber` when a number fails to parse.
+
+### Changed
+- **[BREAKING CHANGE]** `parse` now returns a vector of errors.
+- **[BREAKING CHANGE]** `TokenType::StringLiteral::multi_line` has been replaced with `TokenType::StringLiteral::multi_line_depth`. It serves the same purpose except instead of being an `Option<usize>`, it is now a standard `usize`. It is advised to simply check `quote_type == StringLiteralQuoteType::Brackets` to get the previous behavior.
+- **[BREAKING CHANGE]** Flattened `AstError` into just what used to be `AstError::UnexpectedToken`.
+- **[BREAKING CHANGE]** `Symbol::PlusEqual` and friends are now only available when using Luau.
+- Shebangs provide their trailing trivia more accurately to the rest of full-moon.
+- Attempting to display `StringLiteralQuoteType::Brackets` now returns an error rather than being marked as unreachable.
+- Significantly optimized the entire codebase, helping both time to parse and wasting less stack, especially in debug mode.
+- `Punctuated<T>` now implements `Default` for all `T`, rather than if `T: Default`.
+
+### Removed
+- Removed `UnOp::precedence`, as unary operators do not traditionally use precedence in the same way binary operators do.
+- Removed `TokenizerErrorType::UnexpectedShebang`.
+- Removed `stacker` feature flag, as rewrites to the parser should make it unnecessary.
+
+### Fixed
+- Fixed comments with Unicode characters having positions that report their `character` as bytes.
 
 ## [0.19.0] - 2023-11-10
 ### Added

benches/date.rs

@@ -1,19 +1,11 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use criterion::{criterion_group, criterion_main, Criterion};
 use full_moon::node::Node;
 
 const DATE_SOURCE: &str = include_str!("./date.lua");
 
-fn tokenize(criterion: &mut Criterion) {
-    criterion.bench_function("tokenize date", |b| {
-        b.iter(|| full_moon::tokenizer::tokens(black_box(DATE_SOURCE)))
-    });
-}
-
 fn parse(criterion: &mut Criterion) {
-    let tokens = full_moon::tokenizer::tokens(DATE_SOURCE).unwrap();
-
     criterion.bench_function("get ast from parsed date", move |b| {
-        b.iter(|| full_moon::ast::Ast::from_tokens(black_box(tokens.clone())))
+        b.iter(|| full_moon::parse(DATE_SOURCE))
     });
 }
 
@@ -28,7 +20,7 @@ fn range(criterion: &mut Criterion) {
 criterion_group! {
     name = benches;
     config = Criterion::default().sample_size(20);
-    targets = tokenize, parse, range
+    targets = parse, range
 }
 
 criterion_main!(benches);

benches/t.rs

@@ -1,19 +1,11 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use criterion::{criterion_group, criterion_main, Criterion};
 use full_moon::node::Node;
 
 const T_SOURCE: &str = include_str!("./t.lua");
 
-fn tokenize(criterion: &mut Criterion) {
-    criterion.bench_function("tokenize t", |b| {
-        b.iter(|| full_moon::tokenizer::tokens(black_box(T_SOURCE)))
-    });
-}
-
 fn parse(criterion: &mut Criterion) {
-    let tokens = full_moon::tokenizer::tokens(T_SOURCE).unwrap();
-
     criterion.bench_function("get ast from parsed t", move |b| {
-        b.iter(|| full_moon::ast::Ast::from_tokens(black_box(tokens.clone())))
+        b.iter(|| full_moon::parse(T_SOURCE))
     });
 }
 
@@ -28,7 +20,7 @@ fn range(criterion: &mut Criterion) {
 criterion_group! {
     name = benches;
     config = Criterion::default().sample_size(20);
-    targets = tokenize, parse, range
+    targets = parse, range
 }
 
 criterion_main!(benches);

full-moon/Cargo.toml

@@ -11,13 +11,14 @@ keywords = ["lua", "parser", "lua51", "lua52", "luau"]
 edition = "2021"
 
 [package.metadata.docs.rs]
-# Build Locally: RUSTDOCFLAGS="--cfg doc_cfg" cargo +nightly doc --features roblox,lua52,lua53,lua54  --no-deps --open
-features = ["roblox", "lua52", "lua53", "lua54"]
+# Build Locally: RUSTDOCFLAGS="--cfg doc_cfg" cargo +nightly doc --features luau,lua52,lua53,lua54  --no-deps --open
+features = ["luau", "lua52", "lua53", "lua54"]
 rustdoc-args = ["--cfg", "doc_cfg"]
 
 [features]
 default = ["serde"]
-roblox = []
+luau = ["roblox"]
+roblox = ["luau"]    # backwards compatibility
 lua52 = []
 lua53 = ["lua52"]
 lua54 = ["lua53"]
@@ -28,16 +29,17 @@ bytecount = "0.6"
 cfg-if = "1.0"
 derive_more = "0.99"
 full_moon_derive = { path = "../full-moon-derive", version = "=0.11.0" }
-logos = "0.12.1"
 paste = "1.0"
 serde = { version = "1.0", features = ["derive", "rc"], optional = true }
 smol_str = { version = "0.1.23", features = ["serde"] }
-stacker = { version = "0.1.15", optional = true }
 
 [dev-dependencies]
+codespan = "0.11.1"
+codespan-reporting = "0.11.1"
 criterion = "0.5.1"
 insta = { version = "1.26.0", features = ["glob", "yaml"] }
 pretty_assertions = "1.3.0"
+termcolor = "1.2.0"
 
 [[bench]]
 name = "date"

full-moon/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage

full-moon/fuzz/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "full_moon_fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+old_full_moon = { package = "full_moon", version = "0.18.1", features = [
+	"stacker",
+] }
+
+[dependencies.full_moon]
+path = ".."
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[profile.release]
+debug = 1
+
+[[bin]]
+name = "parse"
+path = "fuzz_targets/parse.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "parse_parity"
+path = "fuzz_targets/parse_parity.rs"
+test = false
+doc = false

full-moon/fuzz/fuzz_targets/parse.rs

@@ -0,0 +1,9 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(string) = std::str::from_utf8(data) {
+        full_moon::parse_fallible(string, full_moon::LuaVersion::new());
+    }
+});

full-moon/fuzz/fuzz_targets/parse_parity.rs

@@ -0,0 +1,11 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(string) = std::str::from_utf8(data) {
+        if old_full_moon::parse(string).is_ok() {
+            full_moon::parse_fallible(string, full_moon::LuaVersion::new());
+        }
+    }
+});