From 53c6a54fb83a722b2de8595831b4a4688c6cc7a9 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 16 Aug 2021 13:21:11 +0200 Subject: [PATCH] lexer post processing - EBNF --- samples/ParserExample/ParserExample.csproj | 4 + samples/ParserExample/Program.cs | 126 +++++++++++++++++- .../expressionModel/BinaryOperation.cs | 56 ++++++++ .../expressionModel/Expression.cs | 9 ++ .../expressionModel/ExpressionContext.cs | 25 ++++ .../expressionModel/FormulaParser.cs | 77 +++++++++++ .../expressionModel/FormulaToken.cs | 44 ++++++ .../expressionModel/FunctionCall.cs | 99 ++++++++++++++ .../ParserExample/expressionModel/Group.cs | 17 +++ .../ParserExample/expressionModel/Number.cs | 17 +++ .../expressionModel/UnaryOperation.cs | 37 +++++ .../ParserExample/expressionModel/Variable.cs | 18 +++ sly/parser/generator/EBNFParserBuilder.cs | 2 +- sly/parser/generator/ParserBuilder.cs | 2 +- sly/sly.csproj | 4 +- 15 files changed, 527 insertions(+), 10 deletions(-) create mode 100644 samples/ParserExample/expressionModel/BinaryOperation.cs create mode 100644 samples/ParserExample/expressionModel/Expression.cs create mode 100644 samples/ParserExample/expressionModel/ExpressionContext.cs create mode 100644 samples/ParserExample/expressionModel/FormulaParser.cs create mode 100644 samples/ParserExample/expressionModel/FormulaToken.cs create mode 100644 samples/ParserExample/expressionModel/FunctionCall.cs create mode 100644 samples/ParserExample/expressionModel/Group.cs create mode 100644 samples/ParserExample/expressionModel/Number.cs create mode 100644 samples/ParserExample/expressionModel/UnaryOperation.cs create mode 100644 samples/ParserExample/expressionModel/Variable.cs diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj index e40db501..66e13cab 100644 --- a/samples/ParserExample/ParserExample.csproj +++ b/samples/ParserExample/ParserExample.csproj @@ -33,6 +33,10 @@ + + + + 1701;1702;1705;1591 diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index 00a1bd8e..81976f1c 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -16,6 +16,7 @@ using indented; using jsonparser; using jsonparser.JsonModel; +using ParserExample.expressionModel; using ParserTests; using ParserTests.Issue239; using ParserTests.lexer; @@ -30,6 +31,8 @@ using sly.parser.generator.visitor; using sly.parser.parser; using Xunit; +using Expression = ParserExample.expressionModel.Expression; +using ExpressionContext = ParserExample.expressionModel.ExpressionContext; namespace ParserExample { @@ -930,7 +933,8 @@ private static void Main(string[] args) // Console.ReadLine(); // TestShortGeneric(); //TestIssue239(); - TestLexerPostProcess(); + //TestLexerPostProcess(); + TestLexerPostProcessEBNF(); } @@ -1010,10 +1014,44 @@ private static List> postProcess(List> postProcessFormula(List> tokens) + { + var mayLeft = new List() + { + FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.IDENTIFIER + }; + + var mayRight = new List() + { + FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.LPAREN, FormulaToken.IDENTIFIER + }; + + Func mayOmmitLeft = (FormulaToken tokenid) => mayLeft.Contains(tokenid); + + Func mayOmmitRight = (FormulaToken tokenid) => mayRight.Contains(tokenid); + + + List> newTokens = new List>(); + for (int i = 0; i < tokens.Count; i++) + { + if ( i >= 1 && + mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID)) + { + newTokens.Add(new Token() + { + TokenID = FormulaToken.TIMES + }); + } + newTokens.Add(tokens[i]); + } + + return newTokens; + } private static void TestLexerPostProcess() { var parserInstance = new VariableExpressionParser(); - var builder = new ParserBuilder(); + var builder = new ParserBuilder(); var build = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression", lexerPostProcess: postProcess); if (build.IsError) @@ -1037,7 +1075,83 @@ private static void TestLexerPostProcess() return; } - var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); + + + r = Parser.Parse("2 x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("2 ( x ) "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("x x "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); + + } + + private static void TestLexerPostProcessEBNF() + { + var parserInstance = new FormulaParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, $"{typeof(FormulaParser).Name}_expressions", + lexerPostProcess: postProcessFormula); + if (build.IsError) + { + foreach (var error in build.Errors) + { + Console.WriteLine(error.Message); + } + + return; + } + + var Parser = build.Result; + var r = Parser.Parse("2 * x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); @@ -1052,7 +1166,7 @@ private static void TestLexerPostProcess() return; } - res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); @@ -1066,7 +1180,7 @@ private static void TestLexerPostProcess() return; } - res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); @@ -1080,7 +1194,7 @@ private static void TestLexerPostProcess() return; } - res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); diff --git a/samples/ParserExample/expressionModel/BinaryOperation.cs b/samples/ParserExample/expressionModel/BinaryOperation.cs new file mode 100644 index 00000000..7e084531 --- /dev/null +++ b/samples/ParserExample/expressionModel/BinaryOperation.cs @@ -0,0 +1,56 @@ +using System; +using expressionparser.model; + +namespace ParserExample.expressionModel +{ + public class BinaryOperation : Expression + { + private readonly Expression LeftExpresion; + private readonly FormulaToken Operator; + private readonly Expression RightExpression; + + + public BinaryOperation(Expression left, FormulaToken op, Expression right) + { + LeftExpresion = left; + Operator = op; + RightExpression = right; + } + + public double? Evaluate(ExpressionContext context) + { + var left = LeftExpresion.Evaluate(context); + var right = RightExpression.Evaluate(context); + + if (left.HasValue && right.HasValue) + switch (Operator) + { + case FormulaToken.PLUS: + { + return left.Value + right.Value; + } + case FormulaToken.MINUS: + { + return left.Value - right.Value; + } + case FormulaToken.TIMES: + { + return left.Value * right.Value; + } + case FormulaToken.DIVIDE: + { + return left.Value / right.Value; + } + case FormulaToken.EXP: + { + return Math.Pow(left.Value,right.Value); + } + default: + { + return null; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Expression.cs b/samples/ParserExample/expressionModel/Expression.cs new file mode 100644 index 00000000..26384cd8 --- /dev/null +++ b/samples/ParserExample/expressionModel/Expression.cs @@ -0,0 +1,9 @@ +using expressionparser.model; + +namespace ParserExample.expressionModel +{ + public interface Expression + { + double? Evaluate(ExpressionContext context); + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/ExpressionContext.cs b/samples/ParserExample/expressionModel/ExpressionContext.cs new file mode 100644 index 00000000..dfee22f4 --- /dev/null +++ b/samples/ParserExample/expressionModel/ExpressionContext.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; + +namespace ParserExample.expressionModel +{ + public class ExpressionContext + { + private readonly Dictionary Variables; + + public ExpressionContext() + { + Variables = new Dictionary(); + } + + public ExpressionContext(Dictionary variables) + { + Variables = variables; + } + + public double? GetValue(string variable) + { + if (Variables.ContainsKey(variable)) return Variables[variable]; + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FormulaParser.cs b/samples/ParserExample/expressionModel/FormulaParser.cs new file mode 100644 index 00000000..eef23c19 --- /dev/null +++ b/samples/ParserExample/expressionModel/FormulaParser.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserExample.expressionModel +{ + public class FormulaParser + { + [Operation((int) FormulaToken.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation((int) FormulaToken.MINUS, Affix.InFix, Associativity.Left, 10)] + public Expression BinaryTermExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + + [Operation((int) FormulaToken.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation((int) FormulaToken.DIVIDE, Affix.InFix, Associativity.Left, 50)] + public Expression BinaryFactorExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + + [Operation((int) FormulaToken.MINUS, Affix.PreFix, Associativity.Right, 100)] + public Expression PreFixExpression(Token operation, Expression value) + { + return new UnaryOperation(FormulaToken.MINUS,value); + } + + [Operation((int) FormulaToken.EXP, Affix.InFix, Associativity.Left, 90)] + public Expression ExpExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + [Operand] + [Production("operand : primary_value")] + public Expression OperandValue(Expression value) + { + return value; + } + + + [Production("primary_value : IDENTIFIER")] + public Expression OperandVariable(Token identifier) + { + return new Variable(identifier.Value); + } + + [Production("primary_value : DOUBLE")] + [Production("primary_value : INT")] + public Expression OperandInt(Token value) + { + return new Number(value.DoubleValue); + } + + [Production("primary_value : LPAREN FormulaParser_expressions RPAREN")] + public Expression OperandParens(Token lparen, Expression value, Token rparen) + { + return value; + } + + [Production( + "primary_value : IDENTIFIER LPAREN[d] FormulaParser_expressions (COMMA FormulaParser_expressions)* RPAREN[d]")] + public Expression FunctionCall(Token funcName, Expression first, + List> others) + { + List parameters = new List(); + parameters.Add(first); + parameters.AddRange(others.Select(x => x.Value(0))); + return new FunctionCall(funcName.Value, parameters); + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FormulaToken.cs b/samples/ParserExample/expressionModel/FormulaToken.cs new file mode 100644 index 00000000..a5c396e3 --- /dev/null +++ b/samples/ParserExample/expressionModel/FormulaToken.cs @@ -0,0 +1,44 @@ +using sly.lexer; + +namespace ParserExample.expressionModel +{ + public enum FormulaToken + { + // float number + [Lexeme(GenericToken.Double)] DOUBLE = 1, + + // integer + [Lexeme(GenericToken.Int)] INT = 3, + + [Lexeme(GenericToken.Identifier)] IDENTIFIER = 4, + + // the + operator + [Lexeme(GenericToken.SugarToken, "+")] PLUS = 5, + + // the ++ operator + [Lexeme(GenericToken.SugarToken, "++")] + INCREMENT = 6, + + // the - operator + [Lexeme(GenericToken.SugarToken, "-")] MINUS = 7, + + // the * operator + [Lexeme(GenericToken.SugarToken, "*")] TIMES = 8, + + // the / operator + [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 9, + + // a left paranthesis ( + [Lexeme(GenericToken.SugarToken, "(")] LPAREN = 10, + + // a right paranthesis ) + [Lexeme(GenericToken.SugarToken, ")")] RPAREN = 11, + + [Sugar(",")]COMMA=12, + // a variable + + [Sugar("^")] EXP=13 + + + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FunctionCall.cs b/samples/ParserExample/expressionModel/FunctionCall.cs new file mode 100644 index 00000000..38499553 --- /dev/null +++ b/samples/ParserExample/expressionModel/FunctionCall.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ParserExample.expressionModel +{ + public class FunctionCall : Expression + { + + public static double Sin(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Sin(parameters.First().Value); + } + + return 0.0; + } + + public static double Cos(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Cos(parameters.First().Value); + } + return 0.0; + } + + public static double Tan(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Tan(parameters.First().Value); + } + return 0.0; + } + + public static double Sqrt(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Sqrt(parameters.First().Value); + } + return 0.0; + } + + public static double Ln(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Log(parameters.First().Value); + } + return 0.0; + } + + public static double Log2(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Log2(parameters.First().Value); + } + return 0.0; + } + + public List Parameters { get; set; } + + public string Name { get; set; } + + public FunctionCall(string name, List parameters) + { + Name = name; + Parameters = parameters; + } + + public double? Evaluate(ExpressionContext context) + { + var parameters = Parameters.Select(x => x.Evaluate(context)); + switch (Name) + { + case "sin" : + return Sin(parameters); + case "cos" : + return Cos(parameters); + case "tan" : + return Tan(parameters); + case "sqrt" : + return Sqrt(parameters); + case "ln" : + return Ln(parameters); + case "log" : + return Log2(parameters); + } + + return 0.0; + } + + + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Group.cs b/samples/ParserExample/expressionModel/Group.cs new file mode 100644 index 00000000..758bfadc --- /dev/null +++ b/samples/ParserExample/expressionModel/Group.cs @@ -0,0 +1,17 @@ +namespace ParserExample.expressionModel +{ + public class Group : ParserExample.expressionModel.Expression + { + private readonly ParserExample.expressionModel.Expression InnerExpression; + + public Group(ParserExample.expressionModel.Expression expr) + { + InnerExpression = expr; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return InnerExpression.Evaluate(context); + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Number.cs b/samples/ParserExample/expressionModel/Number.cs new file mode 100644 index 00000000..e5ed797b --- /dev/null +++ b/samples/ParserExample/expressionModel/Number.cs @@ -0,0 +1,17 @@ +namespace ParserExample.expressionModel +{ + public sealed class Number : ParserExample.expressionModel.Expression + { + private readonly double Value; + + public Number(double value) + { + Value = value; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return Value; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/UnaryOperation.cs b/samples/ParserExample/expressionModel/UnaryOperation.cs new file mode 100644 index 00000000..9c6667af --- /dev/null +++ b/samples/ParserExample/expressionModel/UnaryOperation.cs @@ -0,0 +1,37 @@ +namespace ParserExample.expressionModel +{ + public class UnaryOperation : ParserExample.expressionModel.Expression + { + private readonly FormulaToken Operator; + private readonly ParserExample.expressionModel.Expression RightExpression; + + public UnaryOperation(FormulaToken op, ParserExample.expressionModel.Expression right) + { + Operator = op; + RightExpression = right; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + var right = RightExpression.Evaluate(context); + + if (right.HasValue) + switch (Operator) + { + case FormulaToken.PLUS: + { + return +right.Value; + } + case FormulaToken.MINUS: + { + return -right.Value; + } + default: + { + return null; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Variable.cs b/samples/ParserExample/expressionModel/Variable.cs new file mode 100644 index 00000000..c1c91348 --- /dev/null +++ b/samples/ParserExample/expressionModel/Variable.cs @@ -0,0 +1,18 @@ +namespace ParserExample.expressionModel +{ + public sealed class Variable : ParserExample.expressionModel.Expression + { + private readonly string VariableName; + + public Variable(string varName) + { + VariableName = varName; + } + + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return context.GetValue(VariableName); + } + } +} \ No newline at end of file diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 241bc200..6397bb57 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -66,7 +66,7 @@ public override BuildResult> BuildParser(object parserInstance, visitor = new EBNFSyntaxTreeVisitor(configuration, parserInstance); var parser = new Parser(I18n,syntaxParser, visitor); parser.Configuration = configuration; - var lexerResult = BuildLexer(extensionBuilder); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index f791008e..097a8a9c 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -89,7 +89,7 @@ public virtual BuildResult> BuildParser(object parserInstance, P { var builder = new EBNFParserBuilder(I18n); result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule, - extensionBuilder); + extensionBuilder,lexerPostProcess); } parser = result.Result; diff --git a/sly/sly.csproj b/sly/sly.csproj index 40434f69..39fd71e2 100644 --- a/sly/sly.csproj +++ b/sly/sly.csproj @@ -6,11 +6,11 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 2.7.0.4-alpha + 2.7.0.4-alpha.2 https://github.com/b3b00/sly https://github.com/b3b00/sly https://github.com/b3b00/sly/blob/master/LICENSE - 2.7.0.4-alpha + 2.7.0.4-alpha.2 Library