Merge pull request #502 from b3b00/bugfix/#501

fix #501

src/sly/lexer/GenericLexer.cs

@@ -300,10 +300,6 @@ private FSMLexer<GenericToken> SetLexerMode(Token<IN> token, Stack<FSMLexer<Gene
                     LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
                     lexersStack.Push(LexerFsm);
                 }
-                else
-                {
-                    LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
-                }
             }
 
             return LexerFsm;
@@ -1230,14 +1226,22 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
             var newPush = match.NewPosition.IsPush;
             var newPop = match.NewPosition.IsPop;
             if (inTok != null
-                && derivedTokens.TryGetValue(inTok.TokenID, out var derivations)
-                && derivations.TryGetValue(inTok.Value, out var derivation))
+                && derivedTokens.TryGetValue(inTok.TokenID, out var derivations))
             {
-                newPop = derivation.isPop;
-                newPush = derivation.isPush;
-                newMode = derivation.mode;
+                if (derivations.TryGetValue(inTok.Value, out var derivation))
+                {
+                    newPop = derivation.isPop;
+                    newPush = derivation.isPush;
+                    newMode = derivation.mode;
+                }
+                else
+                {
+                    newPop = false;
+                    newPush = false;
+                    newMode = null;
+                }
             }
-            
+
             tok.IsComment = inTok.IsComment;
             tok.IsEmpty = inTok.IsEmpty;
             tok.SpanValue = inTok.SpanValue;
@@ -1272,5 +1276,22 @@ public string ToGraphViz()
         {
             return TempLexerFsm.ToGraphViz();
         }
+
+        public IList<string> GetSubLexers()
+        {
+            return SubLexersFsm.Keys.ToList();
+        }
+
+        public string ToGraphViz(string subLexerName)
+        {
+            if (SubLexersFsm.TryGetValue(subLexerName, out var subLexer))
+            {
+                return subLexer.ToGraphViz();
+            }
+
+            return null;
+        }
+
+
     }
 }

src/sly/lexer/LexerBuilder.cs

@@ -579,7 +579,7 @@ private static BuildResult<ILexer<IN>> BuildGenericLexer<IN>(IDictionary<IN, Lis
                         {
                             lexer.FSMBuilder.Push(lexeme.Pushtarget);
                         }
-
+                        
                         if (lexeme.IsPop)
                         {
                             lexer.FSMBuilder.Pop();

tests/ParserTests/Issue501/Issue501Tests.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NFluent;
+using sly.lexer;
+using sly.parser;
+using Xunit;
+
+namespace ParserTests.Issue501;
+
+public class Issue501Tests
+{
+    [Fact]
+    public void TestIssue501()
+    {
+        var build = LexerBuilder.BuildLexer<Issue501Token>();
+        Check.That(build).IsOk();
+        Check.That(build.Result).IsNotNull();
+        Check.That(build.Result).IsInstanceOf<GenericLexer<Issue501Token>>();
+        var lexer = build.Result as GenericLexer<Issue501Token>;
+        var source = @"test = 3";
+        var lexed = lexer.Tokenize(source);
+        Check.That(lexed).IsOkLexing();
+        var tokens = lexed.Tokens;
+        Check.That(tokens).IsNotNull();
+        var mainTokens = tokens.MainTokens();
+        Check.That(mainTokens).CountIs(4);
+        Check.That(mainTokens.Last().IsEOS).IsTrue();
+        Check.That(mainTokens.Take(3).Select(x => x.TokenID)).IsEquivalentTo(new List<Issue501Token>()
+            { Issue501Token.Identifier, Issue501Token.Assign, Issue501Token.Number });
+
+    }
+}

tests/ParserTests/Issue501/issue501Token.cs

@@ -0,0 +1,24 @@
+using sly.lexer;
+
+namespace ParserTests.Issue501;
+
+public enum Issue501Token
+{
+    [Keyword("import")]
+    [Push("importMode")]
+    ImportKeyword,
+    [UpTo(";")]
+    [Mode("importMode")]
+    ImportContent,
+    [Sugar(";")]
+    [Mode("importMode")]
+    [Pop]
+    EndImport,
+
+    [AlphaNumDashId]
+    Identifier,
+    [Int]
+    Number,
+    [Sugar("=")]
+    Assign
+}