bugfix #239

ParserTests/Issue239/Issue239Lexer.cs

@@ -1,7 +1,20 @@
+using sly.lexer;
+
 namespace ParserTests.Issue239
 {
     public enum Issue239Lexer
     {
+        [AlphaNumDashId]
+        ID,
+        [Keyword("int")]
+        INT,
+        [Int]
+        INT_LITERAL,
+        [Sugar("=")]
+        ASSIGN,
+        [Sugar(";")]
+        SEMI
+
 
     }
 }

ParserTests/Issue239/Issue239Parser.cs

@@ -1,7 +1,30 @@
+using System.Collections.Generic;
+using sly.lexer;
+using sly.parser.generator;
+
 namespace ParserTests.Issue239
 {
     public class Issue239Parser
     {
+        [Production("statements : statement*")]
+        public object Statements(List<object> statements)
+        {
+            return statements;
+        }
+
+        [Production("statement: INT[d] ID SEMI[d]")]
+        public object IntDeclaration(Token<Issue239Lexer> id)
+        {
+            return $"{id.Value} is an int;\n";
+        }
+
+        [Production("statement : ID ASSIGN[d] INT_LITERAL SEMI[d]")]
+        public object Assignement(Token<Issue239Lexer> id, Token<Issue239Lexer> value)
+        {
+            return $"{id.Value} is equal to {value.IntValue}\n";
+        }
 
+
+
     }
 }

ParserTests/Issue239/Issue239Tests.cs

@@ -1,7 +1,36 @@
+using System.Collections.Generic;
+using expressionparser;
+using simpleExpressionParser;
+using sly.parser;
+using sly.parser.generator;
+using Xunit;
+
 namespace ParserTests.Issue239
 {
     public class Issue239Tests
     {
-
+        private static Parser<Issue239Lexer, object> BuildParser()
+        {   
+            var StartingRule = $"statements";
+            var parserInstance = new Issue239Parser();
+            var builder = new ParserBuilder<Issue239Lexer, object>();
+            var pBuild = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule);
+            Assert.True(pBuild.IsOk);
+            Assert.NotNull(pBuild.Result);
+            return pBuild.Result;
+        }
+
+
+
+        [Fact]
+        public static void TestOk()
+        {
+            var parser = BuildParser();
+            var parseResult = parser.Parse("int x; int y; a = 12;");
+            Assert.True(parseResult.IsOk);
+            Assert.IsAssignableFrom<List<object>>(parseResult.Result);
+            var lst = parseResult.Result as List<object>;
+            Assert.Equal(3, lst.Count);
+        }
     }
 }

ParserTests/lexer/GenericLexerTests.cs

@@ -125,6 +125,13 @@ public enum Extensions
 
         [Lexeme(GenericToken.Double)] DOUBLE
     }
+
+    public enum ShortExtensions
+    {
+        [Extension] DATE,
+
+        [Double] DOUBLE
+    }
 
 
     public class ParserUsingLexerExtensions
@@ -196,6 +203,27 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic
                     .CallBack(callback);
             }
         }
+
+        public static void AddShortExtension(ShortExtensions token, LexemeAttribute lexem, GenericLexer<ShortExtensions> lexer)
+        {
+            if (token == ShortExtensions.DATE)
+            {
+                NodeCallback<GenericToken> callback = match =>
+                {
+                    match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
+                    return match;
+                };
+
+                var fsmBuilder = lexer.FSMBuilder;
+
+                fsmBuilder.GoTo(GenericLexer<Extensions>.in_double)
+                    .Transition('.', CheckDate)
+                    .Mark("start_date")
+                    .RepetitionTransition(4, "[0-9]")
+                    .End(GenericToken.Extension)
+                    .CallBack(callback);
+            }
+        }
     }
 
     public enum CharTokens {
@@ -290,7 +318,8 @@ public enum CustomId
     {
         EOS,
 
-        [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")]
+        [CustomId("A-Za-z", "-_0-9A-Za-z")]
+        // [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")]
         ID,
 
         [Lexeme(GenericToken.SugarToken, "-", "_")]
@@ -775,6 +804,26 @@ public void TestExtensions()
             Assert.Equal("3.14", r.Tokens[1].Value);
 
         }
+
+        [Fact]
+        public void TestShortExtensions()
+        {
+            var lexerRes =
+                LexerBuilder.BuildLexer(new BuildResult<ILexer<ShortExtensions>>(), ExtendedGenericLexer.AddShortExtension);
+            Assert.False(lexerRes.IsError);
+            var lexer = lexerRes.Result as GenericLexer<ShortExtensions>;
+            Assert.NotNull(lexer);
+
+            var r = lexer.Tokenize("20.02.2018 3.14");
+            Assert.True(r.IsOk);
+
+            Assert.Equal(3, r.Tokens.Count);
+            Assert.Equal(ShortExtensions.DATE, r.Tokens[0].TokenID);
+            Assert.Equal("20.02.2018", r.Tokens[0].Value);
+            Assert.Equal(ShortExtensions.DOUBLE, r.Tokens[1].TokenID);
+            Assert.Equal("3.14", r.Tokens[1].Value);
+
+        }
 
         [Fact]
         public void TestExtensionsPreconditionFailure()