From d6547f563cbf4f03822d5e3df0195efe00d6be2b Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 16 Jun 2021 09:55:24 +0200 Subject: [PATCH 1/2] bugfix #239 --- ParserTests/Issue239/Issue239Lexer.cs | 7 + ParserTests/Issue239/Issue239Parser.cs | 7 + ParserTests/Issue239/Issue239Tests.cs | 7 + samples/ParserExample/Program.cs | 10 +- samples/while/parser/WhileTokenGeneric.cs | 120 ++++++++++++++++-- sly/lexer/fsm/FSMLexer.cs | 7 +- .../generator/ParserConfigurationException.cs | 4 - sly/parser/parser/GroupItem.cs | 6 - .../llparser/RecursiveDescentSyntaxParser.cs | 6 +- 9 files changed, 144 insertions(+), 30 deletions(-) create mode 100644 ParserTests/Issue239/Issue239Lexer.cs create mode 100644 ParserTests/Issue239/Issue239Parser.cs create mode 100644 ParserTests/Issue239/Issue239Tests.cs diff --git a/ParserTests/Issue239/Issue239Lexer.cs b/ParserTests/Issue239/Issue239Lexer.cs new file mode 100644 index 00000000..49efbf2a --- /dev/null +++ b/ParserTests/Issue239/Issue239Lexer.cs @@ -0,0 +1,7 @@ +namespace ParserTests.Issue239 +{ + public enum Issue239Lexer + { + + } +} \ No newline at end of file diff --git a/ParserTests/Issue239/Issue239Parser.cs b/ParserTests/Issue239/Issue239Parser.cs new file mode 100644 index 00000000..becf2b97 --- /dev/null +++ b/ParserTests/Issue239/Issue239Parser.cs @@ -0,0 +1,7 @@ +namespace ParserTests.Issue239 +{ + public class Issue239Parser + { + + } +} \ No newline at end of file diff --git a/ParserTests/Issue239/Issue239Tests.cs b/ParserTests/Issue239/Issue239Tests.cs new file mode 100644 index 00000000..f0ce4a95 --- /dev/null +++ b/ParserTests/Issue239/Issue239Tests.cs @@ -0,0 +1,7 @@ +namespace ParserTests.Issue239 +{ + public class Issue239Tests + { + + } +} \ No newline at end of file diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index 2cafb882..b84285fc 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -16,6 +16,7 @@ using jsonparser; using jsonparser.JsonModel; using ParserTests; +using ParserTests.Issue239; using ParserTests.lexer; using simpleExpressionParser; using sly.lexer; @@ -926,7 +927,8 @@ private static void Main(string[] args) // TestIndentedLang(); // TestI18N(); // Console.ReadLine(); - TestShortGeneric(); + // TestShortGeneric(); + TestIssue239(); } @@ -964,6 +966,12 @@ private static void TestShortGeneric() } ; } + + private static void TestIssue239() + { + Issue239Tests.TestOk(); + ; + } } public enum TestGrammarToken diff --git a/samples/while/parser/WhileTokenGeneric.cs b/samples/while/parser/WhileTokenGeneric.cs index 77a1c903..aa32a203 100644 --- a/samples/while/parser/WhileTokenGeneric.cs +++ b/samples/while/parser/WhileTokenGeneric.cs @@ -56,39 +56,133 @@ public enum WhileTokenGeneric #region operators 30 -> 49 - [Lexeme(GenericToken.SugarToken, ">")] GREATER = 30, + [Sugar( ">")] GREATER = 30, - [Lexeme(GenericToken.SugarToken, "<")] LESSER = 31, + [Sugar( "<")] LESSER = 31, - [Lexeme(GenericToken.SugarToken, "==")] + [Sugar( "==")] EQUALS = 32, - [Lexeme(GenericToken.SugarToken, "!=")] + [Sugar( "!=")] DIFFERENT = 33, - [Lexeme(GenericToken.SugarToken, ".")] CONCAT = 34, + [Sugar( ".")] CONCAT = 34, - [Lexeme(GenericToken.SugarToken, ":=")] + [Sugar( ":=")] ASSIGN = 35, - [Lexeme(GenericToken.SugarToken, "+")] PLUS = 36, + [Sugar( "+")] PLUS = 36, - [Lexeme(GenericToken.SugarToken, "-")] MINUS = 37, + [Sugar( "-")] MINUS = 37, - [Lexeme(GenericToken.SugarToken, "*")] TIMES = 38, + [Sugar( "*")] TIMES = 38, - [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 39, + [Sugar( "/")] DIVIDE = 39, #endregion #region sugar 50 -> - [Lexeme(GenericToken.SugarToken, "(")] LPAREN = 50, + [Sugar( "(")] LPAREN = 50, - [Lexeme(GenericToken.SugarToken, ")")] RPAREN = 51, + [Sugar( ")")] RPAREN = 51, - [Lexeme(GenericToken.SugarToken, ";")] SEMICOLON = 52, + [Sugar( ";")] SEMICOLON = 52, + + + EOF = 0 + + #endregion + } + + public enum ShortWhileTokenGeneric + { + #region keywords 0 -> 19 + + [Keyword("IF")] [Keyword("if")] + IF = 1, + + [Keyword("THEN")] [Keyword("then")] + THEN = 2, + + [Keyword("ELSE")] [Keyword("else")] + ELSE = 3, + + [Keyword("WHILE")] [Keyword("while")] + WHILE = 4, + + [Sugar("DO")] [Sugar("do")] + DO = 5, + + [Keyword("SKIP")] [Keyword( "skip")] + SKIP = 6, + + [Keyword( "TRUE")] [Keyword("true")] + TRUE = 7, + + [Keyword( "FALSE")] [Keyword( "false")] + FALSE = 8, + + [Keyword( "NOT")] [Keyword("not")] + NOT = 9, + + [Keyword( "AND")] [Keyword("and")] + AND = 10, + + [Keyword( "OR")] [Keyword("or")] + OR = 11, + + [Keyword( "PRINT")] [Keyword("print")] + PRINT = 12, + + #endregion + + #region literals 20 -> 29 + + [AlphaId] IDENTIFIER = 20, + + [String] STRING = 21, + + [Int] INT = 22, + + #endregion + + #region operators 30 -> 49 + + [Sugar( ">")] GREATER = 30, + + [Sugar( "<")] LESSER = 31, + + [Sugar( "==")] + EQUALS = 32, + + [Sugar( "!=")] + DIFFERENT = 33, + + [Sugar( ".")] CONCAT = 34, + + [Sugar( ":=")] + ASSIGN = 35, + + [Sugar( "+")] PLUS = 36, + + [Sugar( "-")] MINUS = 37, + + + [Sugar( "*")] TIMES = 38, + + [Sugar( "/")] DIVIDE = 39, + + #endregion + + #region sugar 50 -> + + [Sugar( "(")] LPAREN = 50, + + [Sugar( ")")] RPAREN = 51, + + [Sugar( ";")] SEMICOLON = 52, EOF = 0 diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs index e1609e30..959a6992 100644 --- a/sly/lexer/fsm/FSMLexer.cs +++ b/sly/lexer/fsm/FSMLexer.cs @@ -59,9 +59,10 @@ public FSMLexer() public string ToGraphViz() { var dump = new StringBuilder(); - foreach (var transitions in Transitions.Values) - foreach (var transition in transitions) - dump.AppendLine(transition.ToGraphViz(Nodes)); + + foreach (var transition in Transitions.Values.SelectMany(x => x) ) + dump.AppendLine(transition.ToGraphViz(Nodes)); + return dump.ToString(); } diff --git a/sly/parser/generator/ParserConfigurationException.cs b/sly/parser/generator/ParserConfigurationException.cs index a9b4f90e..a0252774 100644 --- a/sly/parser/generator/ParserConfigurationException.cs +++ b/sly/parser/generator/ParserConfigurationException.cs @@ -4,10 +4,6 @@ namespace sly.parser.generator { public class ParserConfigurationException : Exception { - public ParserConfigurationException() : base("unable to configure parser") - { - } - public ParserConfigurationException(string message) : base(message) { } diff --git a/sly/parser/parser/GroupItem.cs b/sly/parser/parser/GroupItem.cs index debe48c3..378c8cd8 100644 --- a/sly/parser/parser/GroupItem.cs +++ b/sly/parser/parser/GroupItem.cs @@ -8,12 +8,6 @@ public class GroupItem { public string Name; - public GroupItem(string name) - { - Name = name; - IsToken = false; - IsValue = false; - } public GroupItem(string name, Token token) { diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs index fec43618..e26cdbef 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs @@ -172,7 +172,7 @@ public SyntaxParseResult Parse(IList> tokens, string startingNonTe var errors = new List>(); var nt = NonTerminals[start]; - var rules = nt.Rules.Where(r => r.PossibleLeadingTokens.Contains(tokens[0].TokenID)).ToList(); + var rules = nt.Rules.Where(r => !tokens[0].IsEOS && r.PossibleLeadingTokens.Contains(tokens[0].TokenID)).ToList(); if (!rules.Any()) { @@ -243,7 +243,7 @@ public virtual SyntaxParseResult Parse(IList> tokens, Rule rul var errors = new List>(); var isError = false; var children = new List>(); - if (rule.PossibleLeadingTokens.Contains(tokens[position].TokenID)) + if (!tokens[position].IsEOS && rule.PossibleLeadingTokens.Contains(tokens[position].TokenID)) if (rule.Clauses != null && rule.Clauses.Count > 0) { children = new List>(); @@ -377,7 +377,7 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, NonTermin var i = 0; var rules = nt.Rules - .Where(r => startPosition < tokens.Count && r.PossibleLeadingTokens.Contains(tokens[startPosition].TokenID) || r.MayBeEmpty) + .Where(r => startPosition < tokens.Count && !tokens[startPosition].IsEOS && r.PossibleLeadingTokens.Contains(tokens[startPosition].TokenID) || r.MayBeEmpty) .ToList(); if (rules.Count == 0 ) From d45d590e564daee4781246d448c61d98d9af811c Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 16 Jun 2021 10:03:52 +0200 Subject: [PATCH 2/2] bugfix #239 --- ParserTests/Issue239/Issue239Lexer.cs | 13 +++++++ ParserTests/Issue239/Issue239Parser.cs | 23 ++++++++++++ ParserTests/Issue239/Issue239Tests.cs | 31 +++++++++++++++- ParserTests/lexer/GenericLexerTests.cs | 51 +++++++++++++++++++++++++- 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/ParserTests/Issue239/Issue239Lexer.cs b/ParserTests/Issue239/Issue239Lexer.cs index 49efbf2a..146c4b78 100644 --- a/ParserTests/Issue239/Issue239Lexer.cs +++ b/ParserTests/Issue239/Issue239Lexer.cs @@ -1,7 +1,20 @@ +using sly.lexer; + namespace ParserTests.Issue239 { public enum Issue239Lexer { + [AlphaNumDashId] + ID, + [Keyword("int")] + INT, + [Int] + INT_LITERAL, + [Sugar("=")] + ASSIGN, + [Sugar(";")] + SEMI + } } \ No newline at end of file diff --git a/ParserTests/Issue239/Issue239Parser.cs b/ParserTests/Issue239/Issue239Parser.cs index becf2b97..f46ff1d8 100644 --- a/ParserTests/Issue239/Issue239Parser.cs +++ b/ParserTests/Issue239/Issue239Parser.cs @@ -1,7 +1,30 @@ +using System.Collections.Generic; +using sly.lexer; +using sly.parser.generator; + namespace ParserTests.Issue239 { public class Issue239Parser { + [Production("statements : statement*")] + public object Statements(List statements) + { + return statements; + } + + [Production("statement: INT[d] ID SEMI[d]")] + public object IntDeclaration(Token id) + { + return $"{id.Value} is an int;\n"; + } + + [Production("statement : ID ASSIGN[d] INT_LITERAL SEMI[d]")] + public object Assignement(Token id, Token value) + { + return $"{id.Value} is equal to {value.IntValue}\n"; + } + + } } \ No newline at end of file diff --git a/ParserTests/Issue239/Issue239Tests.cs b/ParserTests/Issue239/Issue239Tests.cs index f0ce4a95..87f8c4dc 100644 --- a/ParserTests/Issue239/Issue239Tests.cs +++ b/ParserTests/Issue239/Issue239Tests.cs @@ -1,7 +1,36 @@ +using System.Collections.Generic; +using expressionparser; +using simpleExpressionParser; +using sly.parser; +using sly.parser.generator; +using Xunit; + namespace ParserTests.Issue239 { public class Issue239Tests { - + private static Parser BuildParser() + { + var StartingRule = $"statements"; + var parserInstance = new Issue239Parser(); + var builder = new ParserBuilder(); + var pBuild = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); + Assert.True(pBuild.IsOk); + Assert.NotNull(pBuild.Result); + return pBuild.Result; + } + + + + [Fact] + public static void TestOk() + { + var parser = BuildParser(); + var parseResult = parser.Parse("int x; int y; a = 12;"); + Assert.True(parseResult.IsOk); + Assert.IsAssignableFrom>(parseResult.Result); + var lst = parseResult.Result as List; + Assert.Equal(3, lst.Count); + } } } \ No newline at end of file diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs index 3512330f..1c5af689 100644 --- a/ParserTests/lexer/GenericLexerTests.cs +++ b/ParserTests/lexer/GenericLexerTests.cs @@ -125,6 +125,13 @@ public enum Extensions [Lexeme(GenericToken.Double)] DOUBLE } + + public enum ShortExtensions + { + [Extension] DATE, + + [Double] DOUBLE + } public class ParserUsingLexerExtensions @@ -196,6 +203,27 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic .CallBack(callback); } } + + public static void AddShortExtension(ShortExtensions token, LexemeAttribute lexem, GenericLexer lexer) + { + if (token == ShortExtensions.DATE) + { + NodeCallback callback = match => + { + match.Properties[GenericLexer.DerivedToken] = Extensions.DATE; + return match; + }; + + var fsmBuilder = lexer.FSMBuilder; + + fsmBuilder.GoTo(GenericLexer.in_double) + .Transition('.', CheckDate) + .Mark("start_date") + .RepetitionTransition(4, "[0-9]") + .End(GenericToken.Extension) + .CallBack(callback); + } + } } public enum CharTokens { @@ -290,7 +318,8 @@ public enum CustomId { EOS, - [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")] + [CustomId("A-Za-z", "-_0-9A-Za-z")] + // [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "A-Za-z", "-_0-9A-Za-z")] ID, [Lexeme(GenericToken.SugarToken, "-", "_")] @@ -775,6 +804,26 @@ public void TestExtensions() Assert.Equal("3.14", r.Tokens[1].Value); } + + [Fact] + public void TestShortExtensions() + { + var lexerRes = + LexerBuilder.BuildLexer(new BuildResult>(), ExtendedGenericLexer.AddShortExtension); + Assert.False(lexerRes.IsError); + var lexer = lexerRes.Result as GenericLexer; + Assert.NotNull(lexer); + + var r = lexer.Tokenize("20.02.2018 3.14"); + Assert.True(r.IsOk); + + Assert.Equal(3, r.Tokens.Count); + Assert.Equal(ShortExtensions.DATE, r.Tokens[0].TokenID); + Assert.Equal("20.02.2018", r.Tokens[0].Value); + Assert.Equal(ShortExtensions.DOUBLE, r.Tokens[1].TokenID); + Assert.Equal("3.14", r.Tokens[1].Value); + + } [Fact] public void TestExtensionsPreconditionFailure()