diff --git a/src/sly/lexer/GenericLexer.cs b/src/sly/lexer/GenericLexer.cs index 70d6c307..cc8a22e2 100644 --- a/src/sly/lexer/GenericLexer.cs +++ b/src/sly/lexer/GenericLexer.cs @@ -300,10 +300,6 @@ private FSMLexer SetLexerMode(Token token, Stack Transcode(FSMMatch match) var newPush = match.NewPosition.IsPush; var newPop = match.NewPosition.IsPop; if (inTok != null - && derivedTokens.TryGetValue(inTok.TokenID, out var derivations) - && derivations.TryGetValue(inTok.Value, out var derivation)) + && derivedTokens.TryGetValue(inTok.TokenID, out var derivations)) { - newPop = derivation.isPop; - newPush = derivation.isPush; - newMode = derivation.mode; + if (derivations.TryGetValue(inTok.Value, out var derivation)) + { + newPop = derivation.isPop; + newPush = derivation.isPush; + newMode = derivation.mode; + } + else + { + newPop = false; + newPush = false; + newMode = null; + } } - + tok.IsComment = inTok.IsComment; tok.IsEmpty = inTok.IsEmpty; tok.SpanValue = inTok.SpanValue; @@ -1272,5 +1276,22 @@ public string ToGraphViz() { return TempLexerFsm.ToGraphViz(); } + + public IList GetSubLexers() + { + return SubLexersFsm.Keys.ToList(); + } + + public string ToGraphViz(string subLexerName) + { + if (SubLexersFsm.TryGetValue(subLexerName, out var subLexer)) + { + return subLexer.ToGraphViz(); + } + + return null; + } + + } } \ No newline at end of file diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs index 503945fe..d4e10971 100644 --- a/src/sly/lexer/LexerBuilder.cs +++ b/src/sly/lexer/LexerBuilder.cs @@ -579,7 +579,7 @@ private static BuildResult> BuildGenericLexer(IDictionary(); + Check.That(build).IsOk(); + Check.That(build.Result).IsNotNull(); + Check.That(build.Result).IsInstanceOf>(); + var lexer = build.Result as GenericLexer; + var source = @"test = 3"; + var lexed = lexer.Tokenize(source); + Check.That(lexed).IsOkLexing(); + var tokens = lexed.Tokens; + Check.That(tokens).IsNotNull(); + var mainTokens = tokens.MainTokens(); + Check.That(mainTokens).CountIs(4); + Check.That(mainTokens.Last().IsEOS).IsTrue(); + Check.That(mainTokens.Take(3).Select(x => x.TokenID)).IsEquivalentTo(new List() + { Issue501Token.Identifier, Issue501Token.Assign, Issue501Token.Number }); + + } +} \ No newline at end of file diff --git a/tests/ParserTests/Issue501/issue501Token.cs b/tests/ParserTests/Issue501/issue501Token.cs new file mode 100644 index 00000000..7f7ea4ce --- /dev/null +++ b/tests/ParserTests/Issue501/issue501Token.cs @@ -0,0 +1,24 @@ +using sly.lexer; + +namespace ParserTests.Issue501; + +public enum Issue501Token +{ + [Keyword("import")] + [Push("importMode")] + ImportKeyword, + [UpTo(";")] + [Mode("importMode")] + ImportContent, + [Sugar(";")] + [Mode("importMode")] + [Pop] + EndImport, + + [AlphaNumDashId] + Identifier, + [Int] + Number, + [Sugar("=")] + Assign +} \ No newline at end of file