Skip to content

Commit

Permalink
Support z-escape in interpolated strings
Browse files Browse the repository at this point in the history
  • Loading branch information
JohnnyMorganz committed Nov 17, 2024
1 parent 0361d2c commit d47eca5
Show file tree
Hide file tree
Showing 5 changed files with 348 additions and 4 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed
- Fixed `TokenReference::symbol("//=")` returning DoubleSlash + UnexpectedToken when `luau` and `lua53` are enabled together
- Fixed `\z` escapes not parsing in interpolated strings for `luau`

## [1.1.1] - 2024-11-16

Expand Down
15 changes: 14 additions & 1 deletion full-moon/src/tokenizer/interpolated_strings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,25 @@ pub(crate) fn read_interpolated_string_section(
) -> LexerResult<Token> {
let mut characters = Vec::new();
let mut escape = false;
let mut z_escaped = false;

while let Some(character) = lexer.source.current() {
if !escape && matches!(character, '\n' | '\r') {
if !escape && !z_escaped && matches!(character, '\n' | '\r') {
break;
}

assert_eq!(character, lexer.source.next().unwrap());

match (escape, character) {
(true, 'z') => {
escape = true;
z_escaped = true;
characters.push('z');
}

(true, ..) => {
characters.push(character);
z_escaped = true; // support for '\' followed by a new line
escape = false;
}

Expand All @@ -52,6 +60,11 @@ pub(crate) fn read_interpolated_string_section(
escape = true;
}

(false, '\n' | '\r') if z_escaped => {
z_escaped = false;
characters.push(character);
}

(false, ..) if character == '{' => {
lexer.brace_stack.push(BraceType::InterpolatedString);
return LexerResult::Ok(Token {
Expand Down
192 changes: 191 additions & 1 deletion full-moon/tests/roblox_cases/pass/z-escape-string/ast.snap
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
---
source: full-moon/tests/pass_cases.rs
expression: ast.nodes()
input_file: full-moon/tests/roblox_cases/pass/z-escape-string
---
stmts:
- - FunctionCall:
Expand Down Expand Up @@ -178,4 +179,193 @@ stmts:
quote_type: Double
trailing_trivia: []
- ~

- - FunctionCall:
prefix:
Name:
leading_trivia:
- start_position:
bytes: 56
line: 6
character: 1
end_position:
bytes: 57
line: 6
character: 1
token_type:
type: Whitespace
characters: "\n"
token:
start_position:
bytes: 57
line: 7
character: 1
end_position:
bytes: 62
line: 7
character: 6
token_type:
type: Identifier
identifier: print
trailing_trivia: []
suffixes:
- Call:
AnonymousCall:
Parentheses:
parentheses:
tokens:
- leading_trivia: []
token:
start_position:
bytes: 62
line: 7
character: 6
end_position:
bytes: 63
line: 7
character: 7
token_type:
type: Symbol
symbol: (
trailing_trivia: []
- leading_trivia: []
token:
start_position:
bytes: 86
line: 8
character: 12
end_position:
bytes: 87
line: 8
character: 13
token_type:
type: Symbol
symbol: )
trailing_trivia:
- start_position:
bytes: 87
line: 8
character: 13
end_position:
bytes: 88
line: 8
character: 13
token_type:
type: Whitespace
characters: "\n"
arguments:
pairs:
- End:
InterpolatedString:
segments: []
last_string:
leading_trivia: []
token:
start_position:
bytes: 63
line: 7
character: 7
end_position:
bytes: 86
line: 8
character: 12
token_type:
type: InterpolatedString
literal: "testing \\z\n\t twelve"
kind: Simple
trailing_trivia: []
- ~
- - FunctionCall:
prefix:
Name:
leading_trivia:
- start_position:
bytes: 88
line: 9
character: 1
end_position:
bytes: 89
line: 9
character: 1
token_type:
type: Whitespace
characters: "\n"
token:
start_position:
bytes: 89
line: 10
character: 1
end_position:
bytes: 94
line: 10
character: 6
token_type:
type: Identifier
identifier: print
trailing_trivia: []
suffixes:
- Call:
AnonymousCall:
Parentheses:
parentheses:
tokens:
- leading_trivia: []
token:
start_position:
bytes: 94
line: 10
character: 6
end_position:
bytes: 95
line: 10
character: 7
token_type:
type: Symbol
symbol: (
trailing_trivia: []
- leading_trivia: []
token:
start_position:
bytes: 111
line: 11
character: 8
end_position:
bytes: 112
line: 11
character: 9
token_type:
type: Symbol
symbol: )
trailing_trivia:
- start_position:
bytes: 112
line: 11
character: 9
end_position:
bytes: 113
line: 11
character: 9
token_type:
type: Whitespace
characters: "\n"
arguments:
pairs:
- End:
InterpolatedString:
segments: []
last_string:
leading_trivia: []
token:
start_position:
bytes: 95
line: 10
character: 7
end_position:
bytes: 111
line: 11
character: 8
token_type:
type: InterpolatedString
literal: "Hello \\\n\tWorld"
kind: Simple
trailing_trivia: []
- ~
6 changes: 6 additions & 0 deletions full-moon/tests/roblox_cases/pass/z-escape-string/source.lua
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,9 @@ print("testing \z

print("Hello \
World")

print(`testing \z
twelve`)

print(`Hello \
World`)
Loading

0 comments on commit d47eca5

Please sign in to comment.