Merge pull request #240 from b3b00/bugfix/239

bugfix #239
b3b00 · Jun 16, 2021 · 9c9c6f8 · 9c9c6f8
2 parents 021664d + d45d590
commit 9c9c6f8
Show file tree

Hide file tree

Showing 10 changed files with 259 additions and 31 deletions.
diff --git a/ParserTests/Issue239/Issue239Lexer.cs b/ParserTests/Issue239/Issue239Lexer.cs
@@ -0,0 +1,20 @@
+using sly.lexer;
+
+namespace ParserTests.Issue239
+{
+    public enum Issue239Lexer
+    {
+        [AlphaNumDashId]
+        ID,
+        [Keyword("int")]
+        INT,
+        [Int]
+        INT_LITERAL,
+        [Sugar("=")]
+        ASSIGN,
+        [Sugar(";")]
+        SEMI
+
+
+    }
+}
diff --git a/ParserTests/Issue239/Issue239Parser.cs b/ParserTests/Issue239/Issue239Parser.cs
@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace ParserTests.Issue239
+{
+    public class Issue239Parser
+    {
+        [Production("statements : statement*")]
+        public object Statements(List<object> statements)
+        {
+            return statements;
+        }
+
+        [Production("statement: INT[d] ID SEMI[d]")]
+        public object IntDeclaration(Token<Issue239Lexer> id)
+        {
+            return $"{id.Value} is an int;\n";
+        }
+
+        [Production("statement : ID ASSIGN[d] INT_LITERAL SEMI[d]")]
+        public object Assignement(Token<Issue239Lexer> id, Token<Issue239Lexer> value)
+        {
+            return $"{id.Value} is equal to {value.IntValue}\n";
+        }
+
+
+
+    }
+}
diff --git a/ParserTests/Issue239/Issue239Tests.cs b/ParserTests/Issue239/Issue239Tests.cs
@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using expressionparser;
+using simpleExpressionParser;
+using sly.parser;
+using sly.parser.generator;
+using Xunit;
+
+namespace ParserTests.Issue239
+{
+    public class Issue239Tests
+    {
+        private static Parser<Issue239Lexer, object> BuildParser()
+        {   
+            var StartingRule = $"statements";
+            var parserInstance = new Issue239Parser();
+            var builder = new ParserBuilder<Issue239Lexer, object>();
+            var pBuild = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule);
+            Assert.True(pBuild.IsOk);
+            Assert.NotNull(pBuild.Result);
+            return pBuild.Result;
+        }
+
+
+
+        [Fact]
+        public static void TestOk()
+        {
+            var parser = BuildParser();
+            var parseResult = parser.Parse("int x; int y; a = 12;");
+            Assert.True(parseResult.IsOk);
+            Assert.IsAssignableFrom<List<object>>(parseResult.Result);
+            var lst = parseResult.Result as List<object>;
+            Assert.Equal(3, lst.Count);
+        }
+    }
+}
diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs
@@ -125,6 +125,13 @@ public enum Extensions
 
         [Lexeme(GenericToken.Double)] DOUBLE
     }
+
+    public enum ShortExtensions
+    {
+        [Extension] DATE,
+
+        [Double] DOUBLE
+    }
 
 
     public class ParserUsingLexerExtensions
@@ -196,6 +203,27 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic
                     .CallBack(callback);
             }
         }
+
+        public static void AddShortExtension(ShortExtensions token, LexemeAttribute lexem, GenericLexer<ShortExtensions> lexer)
+        {
+            if (token == ShortExtensions.DATE)
+            {
+                NodeCallback<GenericToken> callback = match =>
+                {
+                    match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
+                    return match;
+                };
+
+                var fsmBuilder = lexer.FSMBuilder;
+
+                fsmBuilder.GoTo(GenericLexer<Extensions>.in_double)
+                    .Transition('.', CheckDate)
+                    .Mark("start_date")
+                    .RepetitionTransition(4, "[0-9]")
+                    .End(GenericToken.Extension)
+                    .CallBack(callback);
+            }
+        }
     }
 
     public enum CharTokens {
@@ -290,7 +318,8 @@ public enum CustomId
     {
         EOS,
 
-        [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")]
+        [CustomId("A-Za-z", "-_0-9A-Za-z")]
+        // [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")]
         ID,
 
         [Lexeme(GenericToken.SugarToken, "-", "_")]
@@ -775,6 +804,26 @@ public void TestExtensions()
             Assert.Equal("3.14", r.Tokens[1].Value);
 
         }
+
+        [Fact]
+        public void TestShortExtensions()
+        {
+            var lexerRes =
+                LexerBuilder.BuildLexer(new BuildResult<ILexer<ShortExtensions>>(), ExtendedGenericLexer.AddShortExtension);
+            Assert.False(lexerRes.IsError);
+            var lexer = lexerRes.Result as GenericLexer<ShortExtensions>;
+            Assert.NotNull(lexer);
+
+            var r = lexer.Tokenize("20.02.2018 3.14");
+            Assert.True(r.IsOk);
+
+            Assert.Equal(3, r.Tokens.Count);
+            Assert.Equal(ShortExtensions.DATE, r.Tokens[0].TokenID);
+            Assert.Equal("20.02.2018", r.Tokens[0].Value);
+            Assert.Equal(ShortExtensions.DOUBLE, r.Tokens[1].TokenID);
+            Assert.Equal("3.14", r.Tokens[1].Value);
+
+        }
 
         [Fact]
         public void TestExtensionsPreconditionFailure()

diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs
@@ -16,6 +16,7 @@
 using jsonparser;
 using jsonparser.JsonModel;
 using ParserTests;
+using ParserTests.Issue239;
 using ParserTests.lexer;
 using simpleExpressionParser;
 using sly.lexer;
@@ -926,7 +927,8 @@ private static void Main(string[] args)
             // TestIndentedLang();
             // TestI18N();
             // Console.ReadLine(); 
-            TestShortGeneric();
+            // TestShortGeneric();
+            TestIssue239();
         }
 
 
@@ -964,6 +966,12 @@ private static void TestShortGeneric()
             }
             ;
         }
+
+        private static void TestIssue239()
+        {
+            Issue239Tests.TestOk();
+            ;
+        }
     }
 
     public enum TestGrammarToken

diff --git a/samples/while/parser/WhileTokenGeneric.cs b/samples/while/parser/WhileTokenGeneric.cs
@@ -56,39 +56,133 @@ public enum WhileTokenGeneric
 
         #region operators 30 -> 49
 
-        [Lexeme(GenericToken.SugarToken, ">")] GREATER = 30,
+        [Sugar( ">")] GREATER = 30,
 
-        [Lexeme(GenericToken.SugarToken, "<")] LESSER = 31,
+        [Sugar( "<")] LESSER = 31,
 
-        [Lexeme(GenericToken.SugarToken, "==")]
+        [Sugar( "==")]
         EQUALS = 32,
 
-        [Lexeme(GenericToken.SugarToken, "!=")]
+        [Sugar( "!=")]
         DIFFERENT = 33,
 
-        [Lexeme(GenericToken.SugarToken, ".")] CONCAT = 34,
+        [Sugar( ".")] CONCAT = 34,
 
-        [Lexeme(GenericToken.SugarToken, ":=")]
+        [Sugar( ":=")]
         ASSIGN = 35,
 
-        [Lexeme(GenericToken.SugarToken, "+")] PLUS = 36,
+        [Sugar( "+")] PLUS = 36,
 
-        [Lexeme(GenericToken.SugarToken, "-")] MINUS = 37,
+        [Sugar( "-")] MINUS = 37,
 
 
-        [Lexeme(GenericToken.SugarToken, "*")] TIMES = 38,
+        [Sugar( "*")] TIMES = 38,
 
-        [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 39,
+        [Sugar( "/")] DIVIDE = 39,
 
         #endregion
 
         #region sugar 50 ->
 
-        [Lexeme(GenericToken.SugarToken, "(")] LPAREN = 50,
+        [Sugar( "(")] LPAREN = 50,
 
-        [Lexeme(GenericToken.SugarToken, ")")] RPAREN = 51,
+        [Sugar( ")")] RPAREN = 51,
 
-        [Lexeme(GenericToken.SugarToken, ";")] SEMICOLON = 52,
+        [Sugar( ";")] SEMICOLON = 52,
+
+
+        EOF = 0
+
+        #endregion
+    }
+
+    public enum ShortWhileTokenGeneric
+    {
+        #region keywords 0 -> 19
+
+        [Keyword("IF")] [Keyword("if")]
+        IF = 1,
+
+        [Keyword("THEN")] [Keyword("then")]
+        THEN = 2,
+
+        [Keyword("ELSE")] [Keyword("else")]
+        ELSE = 3,
+
+        [Keyword("WHILE")] [Keyword("while")]
+        WHILE = 4,
+
+        [Sugar("DO")] [Sugar("do")]
+        DO = 5,
+
+        [Keyword("SKIP")] [Keyword( "skip")]
+        SKIP = 6,
+
+        [Keyword( "TRUE")] [Keyword("true")]
+        TRUE = 7,
+
+        [Keyword( "FALSE")] [Keyword( "false")]
+        FALSE = 8,
+
+        [Keyword( "NOT")] [Keyword("not")]
+        NOT = 9,
+
+        [Keyword( "AND")] [Keyword("and")]
+        AND = 10,
+
+        [Keyword( "OR")] [Keyword("or")]
+        OR = 11,
+
+        [Keyword( "PRINT")] [Keyword("print")]
+        PRINT = 12,
+
+        #endregion
+
+        #region literals 20 -> 29
+
+        [AlphaId] IDENTIFIER = 20,
+
+        [String] STRING = 21,
+
+        [Int] INT = 22,
+
+        #endregion
+
+        #region operators 30 -> 49
+
+        [Sugar( ">")] GREATER = 30,
+
+        [Sugar( "<")] LESSER = 31,
+
+        [Sugar( "==")]
+        EQUALS = 32,
+
+        [Sugar( "!=")]
+        DIFFERENT = 33,
+
+        [Sugar( ".")] CONCAT = 34,
+
+        [Sugar( ":=")]
+        ASSIGN = 35,
+
+        [Sugar( "+")] PLUS = 36,
+
+        [Sugar( "-")] MINUS = 37,
+
+
+        [Sugar( "*")] TIMES = 38,
+
+        [Sugar( "/")] DIVIDE = 39,
+
+        #endregion
+
+        #region sugar 50 ->
+
+        [Sugar( "(")] LPAREN = 50,
+
+        [Sugar( ")")] RPAREN = 51,
+
+        [Sugar( ";")] SEMICOLON = 52,
 
 
         EOF = 0

diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs
@@ -59,9 +59,10 @@ public FSMLexer()
         public string ToGraphViz()
         {
             var dump = new StringBuilder();
-            foreach (var transitions in Transitions.Values)
-                foreach (var transition in transitions)
-                    dump.AppendLine(transition.ToGraphViz(Nodes));
+
+            foreach (var transition in Transitions.Values.SelectMany(x => x) )
+                dump.AppendLine(transition.ToGraphViz(Nodes));
+
             return dump.ToString();
         }
 

diff --git a/sly/parser/generator/ParserConfigurationException.cs b/sly/parser/generator/ParserConfigurationException.cs
@@ -4,10 +4,6 @@ namespace sly.parser.generator
 {
     public class ParserConfigurationException : Exception
     {
-        public ParserConfigurationException() : base("unable to configure parser")
-        {
-        }
-
         public ParserConfigurationException(string message) : base(message)
         {
         }

diff --git a/sly/parser/parser/GroupItem.cs b/sly/parser/parser/GroupItem.cs
@@ -8,12 +8,6 @@ public class GroupItem<IN, OUT>
     {
         public string Name;
 
-        public GroupItem(string name)
-        {
-            Name = name;
-            IsToken = false;
-            IsValue = false;
-        }
 
         public GroupItem(string name, Token<IN> token)
         {
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,10 +4,6 @@ namespace sly.parser.generator @@
     {
         public class ParserConfigurationException : Exception
         {
-            public ParserConfigurationException() : base("unable to configure parser")
-            {
-            }
             public ParserConfigurationException(string message) : base(message)
             {
             }
@@ Expand Down @@