From 62d035d52bc4c865f7ce892d6db36b4c4916d948 Mon Sep 17 00:00:00 2001
From: b3b00 <olivier.duhart@gmail.com>
Date: Fri, 8 Nov 2024 10:20:29 +0100
Subject: [PATCH 1/2] fix #501

---
 src/sly/lexer/GenericLexer.cs               | 41 ++++++++++++++++-----
 src/sly/lexer/LexerBuilder.cs               |  2 +-
 tests/ParserTests/Issue501/Issue501Tests.cs | 41 +++++++++++++++++++++
 tests/ParserTests/Issue501/issue501Token.cs | 24 ++++++++++++
 4 files changed, 97 insertions(+), 11 deletions(-)
 create mode 100644 tests/ParserTests/Issue501/Issue501Tests.cs
 create mode 100644 tests/ParserTests/Issue501/issue501Token.cs

diff --git a/src/sly/lexer/GenericLexer.cs b/src/sly/lexer/GenericLexer.cs
index 70d6c307..cc8a22e2 100644
--- a/src/sly/lexer/GenericLexer.cs
+++ b/src/sly/lexer/GenericLexer.cs
@@ -300,10 +300,6 @@ private FSMLexer<GenericToken> SetLexerMode(Token<IN> token, Stack<FSMLexer<Gene
                     LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
                     lexersStack.Push(LexerFsm);
                 }
-                else
-                {
-                    LexerFsm = SubLexersFsm[token.Position.Mode ?? ModeAttribute.DefaultLexerMode];
-                }
             }
 
             return LexerFsm;
@@ -1230,14 +1226,22 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
             var newPush = match.NewPosition.IsPush;
             var newPop = match.NewPosition.IsPop;
             if (inTok != null
-                && derivedTokens.TryGetValue(inTok.TokenID, out var derivations)
-                && derivations.TryGetValue(inTok.Value, out var derivation))
+                && derivedTokens.TryGetValue(inTok.TokenID, out var derivations))
             {
-                newPop = derivation.isPop;
-                newPush = derivation.isPush;
-                newMode = derivation.mode;
+                if (derivations.TryGetValue(inTok.Value, out var derivation))
+                {
+                    newPop = derivation.isPop;
+                    newPush = derivation.isPush;
+                    newMode = derivation.mode;
+                }
+                else
+                {
+                    newPop = false;
+                    newPush = false;
+                    newMode = null;
+                }
             }
-            
+
             tok.IsComment = inTok.IsComment;
             tok.IsEmpty = inTok.IsEmpty;
             tok.SpanValue = inTok.SpanValue;
@@ -1272,5 +1276,22 @@ public string ToGraphViz()
         {
             return TempLexerFsm.ToGraphViz();
         }
+
+        public IList<string> GetSubLexers()
+        {
+            return SubLexersFsm.Keys.ToList();
+        }
+
+        public string ToGraphViz(string subLexerName)
+        {
+            if (SubLexersFsm.TryGetValue(subLexerName, out var subLexer))
+            {
+                return subLexer.ToGraphViz();
+            }
+
+            return null;
+        }
+        
+        
     }
 }
\ No newline at end of file
diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs
index 503945fe..d4e10971 100644
--- a/src/sly/lexer/LexerBuilder.cs
+++ b/src/sly/lexer/LexerBuilder.cs
@@ -579,7 +579,7 @@ private static BuildResult<ILexer<IN>> BuildGenericLexer<IN>(IDictionary<IN, Lis
                         {
                             lexer.FSMBuilder.Push(lexeme.Pushtarget);
                         }
-
+                        
                         if (lexeme.IsPop)
                         {
                             lexer.FSMBuilder.Pop();
diff --git a/tests/ParserTests/Issue501/Issue501Tests.cs b/tests/ParserTests/Issue501/Issue501Tests.cs
new file mode 100644
index 00000000..9ee896ae
--- /dev/null
+++ b/tests/ParserTests/Issue501/Issue501Tests.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NFluent;
+using sly.lexer;
+using sly.parser;
+using Xunit;
+
+namespace ParserTests.Issue501;
+
+public class Issue501Tests
+{
+    [Fact]
+    public void TestIssue501()
+    {
+        var build = LexerBuilder.BuildLexer<Issue501Token>();
+        Check.That(build).IsOk();
+        Check.That(build.Result).IsNotNull();
+        Check.That(build.Result).IsInstanceOf<GenericLexer<Issue501Token>>();
+        var lexer = build.Result as GenericLexer<Issue501Token>;
+        foreach (var subLexer in lexer.GetSubLexers())
+        {
+            var graph = lexer.ToGraphViz(subLexer);
+            File.WriteAllText(Path.Combine("c:/tmp",$"{subLexer}.txt"),graph);
+        }
+        var source = @"test = 3";
+        var lexed = lexer.Tokenize(source);
+        Check.That(lexed).IsOkLexing();
+        // Check.That(lexed.Error.ErrorType).IsEqualTo(ErrorType.UnexpectedChar);
+        // Check.That(lexed.Error.UnexpectedChar).IsEqualTo('=');
+        var tokens = lexed.Tokens;
+        Check.That(tokens).IsNotNull();
+        var mainTokens = tokens.MainTokens();
+        Check.That(mainTokens).CountIs(4);
+        Check.That(mainTokens.Last().IsEOS).IsTrue();
+        Check.That(mainTokens.Take(3).Select(x => x.TokenID)).IsEquivalentTo(new List<Issue501Token>()
+            { Issue501Token.Identifier, Issue501Token.Assign, Issue501Token.Number });
+        
+    }
+}
\ No newline at end of file
diff --git a/tests/ParserTests/Issue501/issue501Token.cs b/tests/ParserTests/Issue501/issue501Token.cs
new file mode 100644
index 00000000..7f7ea4ce
--- /dev/null
+++ b/tests/ParserTests/Issue501/issue501Token.cs
@@ -0,0 +1,24 @@
+using sly.lexer;
+
+namespace ParserTests.Issue501;
+
+public enum Issue501Token
+{
+    [Keyword("import")]
+    [Push("importMode")]
+    ImportKeyword,
+    [UpTo(";")]
+    [Mode("importMode")]
+    ImportContent,
+    [Sugar(";")]
+    [Mode("importMode")]
+    [Pop]
+    EndImport,
+
+    [AlphaNumDashId]
+    Identifier,
+    [Int]
+    Number,
+    [Sugar("=")]
+    Assign
+}
\ No newline at end of file

From 4e11e2693d4e5d74d18928e085c936feb02a673b Mon Sep 17 00:00:00 2001
From: b3b00 <olivier.duhart@gmail.com>
Date: Fri, 8 Nov 2024 10:27:08 +0100
Subject: [PATCH 2/2] fix #501 unit test

---
 tests/ParserTests/Issue501/Issue501Tests.cs | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/ParserTests/Issue501/Issue501Tests.cs b/tests/ParserTests/Issue501/Issue501Tests.cs
index 9ee896ae..9509a3ab 100644
--- a/tests/ParserTests/Issue501/Issue501Tests.cs
+++ b/tests/ParserTests/Issue501/Issue501Tests.cs
@@ -19,16 +19,9 @@ public void TestIssue501()
         Check.That(build.Result).IsNotNull();
         Check.That(build.Result).IsInstanceOf<GenericLexer<Issue501Token>>();
         var lexer = build.Result as GenericLexer<Issue501Token>;
-        foreach (var subLexer in lexer.GetSubLexers())
-        {
-            var graph = lexer.ToGraphViz(subLexer);
-            File.WriteAllText(Path.Combine("c:/tmp",$"{subLexer}.txt"),graph);
-        }
         var source = @"test = 3";
         var lexed = lexer.Tokenize(source);
         Check.That(lexed).IsOkLexing();
-        // Check.That(lexed.Error.ErrorType).IsEqualTo(ErrorType.UnexpectedChar);
-        // Check.That(lexed.Error.UnexpectedChar).IsEqualTo('=');
         var tokens = lexed.Tokens;
         Check.That(tokens).IsNotNull();
         var mainTokens = tokens.MainTokens();