From b2861fb49a881c3b8b906cdd0149b7500e266a23 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Sun, 15 Mar 2020 18:33:38 +0100 Subject: [PATCH 1/5] looking at #165 --- samples/ParserExample/Issue165Lexer.cs | 50 ++++++++++++++ samples/ParserExample/Issue165Parser.cs | 88 +++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 samples/ParserExample/Issue165Lexer.cs create mode 100644 samples/ParserExample/Issue165Parser.cs diff --git a/samples/ParserExample/Issue165Lexer.cs b/samples/ParserExample/Issue165Lexer.cs new file mode 100644 index 00000000..797f735d --- /dev/null +++ b/samples/ParserExample/Issue165Lexer.cs @@ -0,0 +1,50 @@ +using sly.lexer; + +namespace ParserExample +{ + public enum Issue165Lexer + { + [Lexeme(GenericToken.SugarToken,"?")] + QUESTIONMARK = 1, + + [Lexeme(GenericToken.SugarToken,"+")] + PLUS = 2, + + [Lexeme(GenericToken.SugarToken,"-")] + MINUS = 3, + + [Lexeme(GenericToken.SugarToken,"*")] + TIMES = 4, + + [Lexeme(GenericToken.SugarToken,"/")] + DIVIDE = 5, + + [Lexeme(GenericToken.SugarToken,"(")] + LPAR = 6, + + [Lexeme(GenericToken.SugarToken,")")] + RPAR = 6, + + [Lexeme(GenericToken.SugarToken,"[")] + LBR = 7, + + [Lexeme(GenericToken.SugarToken,"]")] + RBR = 8, + + [Lexeme(GenericToken.SugarToken,":")] + COLON = 7, + + [Lexeme(GenericToken.SugarToken,"=")] + EQ = 8, + + + [Lexeme(GenericToken.Identifier)] ID = 20, + + [Lexeme(GenericToken.String)] STRING = 21, + + [Lexeme(GenericToken.Int)] INT = 22, + + + + } +} \ No newline at end of file diff --git a/samples/ParserExample/Issue165Parser.cs b/samples/ParserExample/Issue165Parser.cs new file mode 100644 index 00000000..d0864c88 --- /dev/null +++ b/samples/ParserExample/Issue165Parser.cs @@ -0,0 +1,88 @@ +using expressionparser; +using sly.lexer; +using sly.parser.generator; + +namespace ParserExample +{ + public class Issue165Parser + { + + [Production("test_stmt : ID EQ [d] arith_expr ")] + public object assign(Token id, object expr) + { + return null; + } + + + + [Operation((int) Issue165Lexer.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation("MINUS", Affix.InFix, Associativity.Left, 10)] + public object BinaryTermExpression(object left, Token operation, object right) + { + return null; + } + + + [Operation((int) Issue165Lexer.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] + public object BinaryFactorExpression(double left, Token operation, object right) + { + + return null; + } + + + + [Production("arith_expr : Issue165Parser_expressions")] + [Production("arith_expr : ternary_expr")] + public object arith(object h) + { + return h; + } + + [Production( + "ternary_expr : Issue165Parser_expressions QUESTIONMARK [d] Issue165Parser_expressions COLON [d] Issue165Parser_expressions")] + public object ternary(object j, object k, object l) + { + return null; + } + + [Production("sqbr_expr : ID LBR [d] Issue165Parser_expressions RBR [d]")] + public object sqrbrack(Token token, object expr) + { + return null; + } + + [Production("group_expr : LPAR [d] Issue165Parser_expressions RPAR [d]")] + public object group(object subexpr) + { + return subexpr; + } + + [Operand] + [Production("atom : ID")] + public object id(Token i) + { + return null; + } + [Operand] + [Production("atom : INT")] + public object integer(Token i) + { + return i; + } + [Operand] + [Production("atom : sqbr_expr")] + public object sq(object i) + { + return i; + } + [Operand] + [Production("atom : group_expr")] + public object id(object g) + { + return g; + } + + } +} \ No newline at end of file From 139b7cc5b4895e7624e64c162c88225c70af1996 Mon Sep 17 00:00:00 2001 From: Nathan Adams Date: Wed, 1 Apr 2020 13:24:46 -0400 Subject: [PATCH 2/5] Optimized Expression Rules for #165 --- .../generator/ExpressionRulesGenerator.cs | 111 ++++++++++-------- sly/parser/syntax/grammar/Rule.cs | 11 +- 2 files changed, 67 insertions(+), 55 deletions(-) diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index 39acc4b3..70158eb1 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -54,18 +54,16 @@ public static BuildResult> BuildExpressionRules>>(); - methods.ForEach(m => { var attributes = - (OperationAttribute[]) m.GetCustomAttributes(typeof(OperationAttribute), true); + (OperationAttribute[])m.GetCustomAttributes(typeof(OperationAttribute), true); foreach (var attr in attributes) { - IN oper = default(IN); + IN oper = default; if (attr.IsIntToken) { oper = EnumConverter.ConvertIntToEnum(attr.IntToken); @@ -74,7 +72,7 @@ public static BuildResult> BuildExpressionRules(attr.StringToken); } - var operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix,oper); + var operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); var operations = new List>(); if (operationsByPrecedence.ContainsKey(operation.Precedence)) operations = operationsByPrecedence[operation.Precedence]; @@ -127,7 +125,6 @@ private static void GenerateExpressionParser(ParserConfiguration(ParserConfiguration(ParserConfiguration BuildNonTerminal(bool last, string name, string nextName, - List> operations, Dictionary>> operationsByPrecedence) - where IN : struct + private static NonTerminal BuildNonTerminal(string name, string nextName, List> operations) where IN : struct { var nonTerminal = new NonTerminal(name, new List>()); - foreach (var operation in operations) - if (operation.Affix == Affix.InFix) + + var InFixOps = operations.Where(x => x.Affix == Affix.InFix).ToList(); + if (InFixOps.Count > 0) + { + var InFixClauses = InFixOps.Select(x => new TerminalClause(x.OperatorToken)).ToList>(); + + var rule = new Rule() { - var rule = new Rule(); - rule.Clauses.Add(new NonTerminalClause(nextName)); - rule.Clauses.Add(new TerminalClause(operation.OperatorToken)); - rule.Clauses.Add(new NonTerminalClause(name)); - rule.IsExpressionRule = true; - rule.ExpressionAffix = operation.Affix; - rule.SetVisitor(operation); - nonTerminal.Rules.Add(rule); - } - else if (operation.Affix == Affix.PreFix) + ExpressionAffix = Affix.InFix, + IsExpressionRule = true + }; + + rule.Clauses.Add(new NonTerminalClause(nextName)); + rule.Clauses.Add(InFixClauses.Count == 1 ? InFixClauses[0] : new ChoiceClause(InFixClauses)); + rule.Clauses.Add(new NonTerminalClause(name)); + + InFixOps.ForEach(x => rule.SetVisitor(x)); + nonTerminal.Rules.Add(rule); + } + + + var PreFixOps = operations.Where(x => x.Affix == Affix.PreFix).ToList(); + if (PreFixOps.Count > 0) + { + var PreFixClauses = PreFixOps.Select(x => new TerminalClause(x.OperatorToken)).ToList>(); + + var rule = new Rule() { - var rule = new Rule(); - rule.Clauses.Add(new TerminalClause(operation.OperatorToken)); - rule.Clauses.Add(new NonTerminalClause(nextName)); - rule.IsExpressionRule = true; - rule.ExpressionAffix = operation.Affix; - rule.SetVisitor(operation); - nonTerminal.Rules.Add(rule); - } - else if (operation.Affix == Affix.PostFix) + ExpressionAffix = Affix.PreFix, + IsExpressionRule = true + }; + + rule.Clauses.Add(PreFixClauses.Count == 1 ? PreFixClauses[0] : new ChoiceClause(PreFixClauses)); + rule.Clauses.Add(new NonTerminalClause(nextName)); + + PreFixOps.ForEach(x => rule.SetVisitor(x)); + nonTerminal.Rules.Add(rule); + } + + var PostFixOps = operations.Where(x => x.Affix == Affix.PostFix).ToList(); + if (PostFixOps.Count > 0) + { + var PostFixClauses = PostFixOps.Select(x => new TerminalClause(x.OperatorToken)).ToList>(); + + var rule = new Rule() { - var rule = new Rule(); - rule.Clauses.Add(new NonTerminalClause(nextName)); - rule.Clauses.Add(new TerminalClause(operation.OperatorToken)); - rule.IsExpressionRule = true; - rule.ExpressionAffix = operation.Affix; - rule.SetVisitor(operation); - nonTerminal.Rules.Add(rule); - } + ExpressionAffix = Affix.PostFix, + IsExpressionRule = true + }; + + rule.Clauses.Add(new NonTerminalClause(nextName)); + rule.Clauses.Add(PostFixClauses.Count == 1 ? PostFixClauses[0] : new ChoiceClause(PostFixClauses)); + + PostFixOps.ForEach(x => rule.SetVisitor(x)); + nonTerminal.Rules.Add(rule); + } var rule0 = new Rule(); rule0.Clauses.Add(new NonTerminalClause(nextName)); @@ -224,13 +240,8 @@ private static string GetNonTerminalNameForPrecedence(int precedence, List oper.ToString()) .ToList() .Aggregate((s1, s2) => $"{s1}_{s2}"); - var name = $"expr_{precedence}_{operatorsPart}"; - - - return name; + + return $"expr_{precedence}_{operatorsPart}"; } - - - } } \ No newline at end of file diff --git a/sly/parser/syntax/grammar/Rule.cs b/sly/parser/syntax/grammar/Rule.cs index f72d20f4..d4a7a110 100644 --- a/sly/parser/syntax/grammar/Rule.cs +++ b/sly/parser/syntax/grammar/Rule.cs @@ -33,11 +33,12 @@ public string Key { get { - var k = Clauses - .Select(c => c.ToString()) - .Aggregate((c1, c2) => c1.ToString() + "_" + c2.ToString()); - if (Clauses.Count == 1) k += "_"; - return k; + var key = string.Join("_", Clauses.Select(c => c.ToString())); + + if (Clauses.Count == 1) + key += "_"; + + return IsExpressionRule ? key.Replace(" | ", "_") : key; } } From 9829e69ca8baf3948e1e5eefc229479dc5016031 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 15 Apr 2020 14:45:59 +0200 Subject: [PATCH 3/5] #165 fix issues after @CP3088 pull-request --- ParserTests/EBNFTests.cs | 2 +- ParserTests/ExpressionGeneratorTests.cs | 6 +++--- sly/parser/generator/ExpressionRulesGenerator.cs | 6 +++++- sly/parser/syntax/tree/SyntaxNode.cs | 6 +++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ParserTests/EBNFTests.cs b/ParserTests/EBNFTests.cs index 9635ff41..c316d45b 100644 --- a/ParserTests/EBNFTests.cs +++ b/ParserTests/EBNFTests.cs @@ -777,7 +777,7 @@ public void TestZeroOrMoreWithOne() #region CONTEXTS - private BuildResult> buildSimpleExpressionParserWithContext(ParserType parserType = ParserType.LL_RECURSIVE_DESCENT) + private BuildResult> buildSimpleExpressionParserWithContext(ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT) { var startingRule = $"{typeof(SimpleExpressionParserWithContext).Name}_expressions"; var parserInstance = new SimpleExpressionParserWithContext(); diff --git a/ParserTests/ExpressionGeneratorTests.cs b/ParserTests/ExpressionGeneratorTests.cs index 7df9fd1e..0930f229 100644 --- a/ParserTests/ExpressionGeneratorTests.cs +++ b/ParserTests/ExpressionGeneratorTests.cs @@ -20,7 +20,7 @@ private void BuildParser() StartingRule = $"{typeof(SimpleExpressionParser).Name}_expressions"; var parserInstance = new SimpleExpressionParser(); var builder = new ParserBuilder(); - Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, StartingRule); + Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); } [Fact] @@ -76,12 +76,12 @@ public void TestBuild() Assert.Equal(3, nt.Rules.Count); Assert.Contains("primary_value", nt.Name); nt = nonterminals[2]; - Assert.Equal(3, nt.Rules.Count); + Assert.Equal(2, nt.Rules.Count); Assert.Contains("10", nt.Name); Assert.Contains("PLUS", nt.Name); Assert.Contains("MINUS", nt.Name); nt = nonterminals[3]; - Assert.Equal(3, nt.Rules.Count); + Assert.Equal(2, nt.Rules.Count); Assert.Contains("50", nt.Name); Assert.Contains("TIMES", nt.Name); Assert.Contains("DIVIDE", nt.Name); diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index 70158eb1..f70d7463 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -171,7 +171,11 @@ private static NonTerminal BuildNonTerminal(string name, string nextName rule.Clauses.Add(InFixClauses.Count == 1 ? InFixClauses[0] : new ChoiceClause(InFixClauses)); rule.Clauses.Add(new NonTerminalClause(name)); - InFixOps.ForEach(x => rule.SetVisitor(x)); + InFixOps.ForEach(x => + { + rule.SetVisitor(x); + rule.IsExpressionRule = true; + }); nonTerminal.Rules.Add(rule); } diff --git a/sly/parser/syntax/tree/SyntaxNode.cs b/sly/parser/syntax/tree/SyntaxNode.cs index 8267ac79..a8686f3b 100644 --- a/sly/parser/syntax/tree/SyntaxNode.cs +++ b/sly/parser/syntax/tree/SyntaxNode.cs @@ -10,7 +10,7 @@ public class SyntaxNode : ISyntaxNode where IN : struct public SyntaxNode(string name, List> children = null, MethodInfo visitor = null) { Name = name; - Children = children == null ? new List>() : children; + Children = children ?? new List>(); Visitor = visitor; } @@ -34,8 +34,8 @@ public SyntaxNode(string name, List> children = null, MethodInfo public bool IsExpressionNode => Operation != null; - public bool IsBinaryOperationNode => IsExpressionNode ? Operation.Affix == Affix.InFix : false; - public bool IsUnaryOperationNode => IsExpressionNode ? Operation.Affix != Affix.InFix : false; + public bool IsBinaryOperationNode => IsExpressionNode && Operation.Affix == Affix.InFix; + public bool IsUnaryOperationNode => IsExpressionNode && Operation.Affix != Affix.InFix; public int Precedence => IsExpressionNode ? Operation.Precedence : -1; public Associativity Associativity => From ceff6a52168cee6658a61619d0fa771b8bae304c Mon Sep 17 00:00:00 2001 From: Olivier Duhart <1224790+b3b00@users.noreply.github.com> Date: Wed, 15 Apr 2020 15:19:54 +0200 Subject: [PATCH 4/5] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 665dd398..3640222f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # C# Lex Yacc # [![Build status](https://ci.appveyor.com/api/projects/status/n9uffgkqn2qet7k9?svg=true)](https://ci.appveyor.com/project/OlivierDuhart/sly) -![AppVeyor tests (compact)](https://img.shields.io/appveyor/tests/OlivierDuhart/sly.svg?compact_message) + +![Test status](http://teststatusbadge.azurewebsites.net/api/status/mmaitre314/securestringcodegen) [![coveralls](https://coveralls.io/repos/github/b3b00/csly/badge.svg?branch=dev)](https://coveralls.io/github/b3b00/csly?branch=dev) ![.NET Core](https://github.com/b3b00/csly/workflows/.NET%20Core/badge.svg) [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fb3b00%2Fcsly.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fb3b00%2Fcsly?ref=badge_shield) From c8f884330371270be00d3df813c65f70d272ab50 Mon Sep 17 00:00:00 2001 From: Olivier Duhart <1224790+b3b00@users.noreply.github.com> Date: Wed, 15 Apr 2020 15:20:52 +0200 Subject: [PATCH 5/5] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 3640222f..e11e8c93 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # C# Lex Yacc # [![Build status](https://ci.appveyor.com/api/projects/status/n9uffgkqn2qet7k9?svg=true)](https://ci.appveyor.com/project/OlivierDuhart/sly) - ![Test status](http://teststatusbadge.azurewebsites.net/api/status/mmaitre314/securestringcodegen) [![coveralls](https://coveralls.io/repos/github/b3b00/csly/badge.svg?branch=dev)](https://coveralls.io/github/b3b00/csly?branch=dev) ![.NET Core](https://github.com/b3b00/csly/workflows/.NET%20Core/badge.svg)