diff --git a/ParserTests/comments/CommentsErrorTest.cs b/ParserTests/comments/CommentsErrorTest.cs index a967ba0b..039f566f 100644 --- a/ParserTests/comments/CommentsErrorTest.cs +++ b/ParserTests/comments/CommentsErrorTest.cs @@ -184,7 +184,7 @@ public void RedundantAttributes() [Fact] public void MixedErrors() { - var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en"); + var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en"); Assert.True(lexerRes10.IsError); Assert.Equal(4, lexerRes10.Errors.Count); var expectedErrors = new[] @@ -199,7 +199,7 @@ public void MixedErrors() expectedErrors = new[] {"too many multi-line comment lexem", "too many single-line comment lexem"}; - var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en"); + var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en"); Assert.True(lexerRes9.IsError); Assert.Equal(2, lexerRes9.Errors.Count); foreach (var expectedError in expectedErrors) @@ -209,7 +209,7 @@ public void MixedErrors() expectedErrors = new[] {"too many multi-line comment lexem","comment lexem can't be used together with single-line or multi-line comment lexems"}; - var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en"); + var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en"); Assert.True(lexerRes8.IsError); Assert.Equal(2, lexerRes8.Errors.Count); foreach (var expectedError in expectedErrors) diff --git a/samples/Issue244/Issue244Parser.cs b/samples/Issue244/Issue244Parser.cs new file mode 100644 index 00000000..a55f57b2 --- /dev/null +++ b/samples/Issue244/Issue244Parser.cs @@ -0,0 +1,152 @@ +using System; +using expressionparser; +using simpleExpressionParser; +using sly.lexer; +using sly.parser.generator; + +namespace Issue244 +{ + public class Issue244Parser + { + [Operation((int) SimpleExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation("MINUS", Affix.InFix, Associativity.Left, 10)] + public Result BinaryTermExpression(Result left, Token operation, Result right) + { + // Get the length of the expression : + int length = right.EndIndex - left.StartIndex; + + + Result result = null; + switch (operation.TokenID) + { + case SimpleExpressionToken.PLUS: + { + result = new Result() + { + Value = left.Value + right.Value, + StartColumn = left.StartColumn, + EndColumn = right.EndColumn, + StartIndex = left.StartIndex, + EndIndex = right.EndIndex + } + ; + break; + } + case SimpleExpressionToken.MINUS: + { + result = new Result() + { + Value = left.Value - right.Value, + StartColumn = left.StartColumn, + EndColumn = right.EndColumn, + StartIndex = left.StartIndex, + EndIndex = right.EndIndex + }; + break; + } + } + + return result; + } + + + [Operation((int) SimpleExpressionToken.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] + public Result BinaryFactorExpression(Result left, Token operation, Result right) + { + Result result = null; + switch (operation.TokenID) + { + case SimpleExpressionToken.TIMES: + { + result = new Result() + { + Value = left.Value * right.Value, + StartColumn = left.StartColumn, + EndColumn = right.EndColumn, + StartIndex = left.StartIndex, + EndIndex = right.EndIndex + }; + break; + } + case SimpleExpressionToken.DIVIDE: + { + result = new Result() + { + Value = left.Value / right.Value, + StartColumn = left.StartColumn, + EndColumn = right.EndColumn, + StartIndex = left.StartIndex, + EndIndex = right.EndIndex + }; + break; + } + } + + return result; + } + + + [Operation((int) SimpleExpressionToken.MINUS, Affix.PreFix, Associativity.Right, 100)] + public Result PreFixExpression(Token operation, Result value) + { + return new Result() + { + Value = -value.Value, + StartColumn = operation.Position.Column, + EndColumn = value.EndColumn, + StartIndex = operation.Position.Index, + EndIndex = value.EndIndex + }; + } + + + + [Operand] + [Production("operand : primary_value")] + public Result OperandValue(Result value) + { + return value; + } + + + [Production("primary_value : DOUBLE")] + public Result OperandDouble(Token value) + { + return new Result() + { + Value = value.DoubleValue, + StartColumn = value.Position.Column, + EndColumn = value.Position.Column + value.Value.Length, + StartIndex = value.Position.Index, + EndIndex = value.Position.Index + value.Value.Length + }; + } + + [Production("primary_value : INT")] + public Result OperandInt(Token value) + { + return new Result() + { + Value = value.DoubleValue, + StartColumn = value.Position.Column, + EndColumn = value.Position.Column + value.Value.Length, + StartIndex = value.Position.Index, + EndIndex = value.Position.Index + value.Value.Length + }; + } + + [Production("primary_value : LPAREN Issue244Parser_expressions RPAREN")] + public Result OperandParens(Token lparen, Result value, Token rparen) + { + return new Result() + { + Value = value.Value, + StartColumn = lparen.Position.Column, + EndColumn = rparen.Position.Column + rparen.Value.Length, + StartIndex = rparen.Position.Index, + EndIndex = rparen.Position.Index + rparen.Value.Length, + }; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj index e40db501..66e13cab 100644 --- a/samples/ParserExample/ParserExample.csproj +++ b/samples/ParserExample/ParserExample.csproj @@ -33,6 +33,10 @@ + + + + 1701;1702;1705;1591 diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index b84285fc..81976f1c 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -11,10 +11,12 @@ using csly.whileLang.model; using csly.whileLang.parser; using expressionparser; +using expressionparser.model; using GenericLexerWithCallbacks; using indented; using jsonparser; using jsonparser.JsonModel; +using ParserExample.expressionModel; using ParserTests; using ParserTests.Issue239; using ParserTests.lexer; @@ -29,6 +31,8 @@ using sly.parser.generator.visitor; using sly.parser.parser; using Xunit; +using Expression = ParserExample.expressionModel.Expression; +using ExpressionContext = ParserExample.expressionModel.ExpressionContext; namespace ParserExample { @@ -928,7 +932,9 @@ private static void Main(string[] args) // TestI18N(); // Console.ReadLine(); // TestShortGeneric(); - TestIssue239(); + //TestIssue239(); + //TestLexerPostProcess(); + TestLexerPostProcessEBNF(); } @@ -972,6 +978,227 @@ private static void TestIssue239() Issue239Tests.TestOk(); ; } + + + private static List> postProcess(List> tokens) + { + var mayLeft = new List() + { + ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.IDENTIFIER + }; + + var mayRight = new List() + { + ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.LPAREN, ExpressionToken.IDENTIFIER + }; + + Func mayOmmitLeft = (ExpressionToken tokenid) => mayLeft.Contains(tokenid); + + Func mayOmmitRight = (ExpressionToken tokenid) => mayRight.Contains(tokenid); + + + List> newTokens = new List>(); + for (int i = 0; i < tokens.Count; i++) + { + if ( i >= 1 && + mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID)) + { + newTokens.Add(new Token() + { + TokenID = ExpressionToken.TIMES + }); + } + newTokens.Add(tokens[i]); + } + + return newTokens; + } + + + private static List> postProcessFormula(List> tokens) + { + var mayLeft = new List() + { + FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.IDENTIFIER + }; + + var mayRight = new List() + { + FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.LPAREN, FormulaToken.IDENTIFIER + }; + + Func mayOmmitLeft = (FormulaToken tokenid) => mayLeft.Contains(tokenid); + + Func mayOmmitRight = (FormulaToken tokenid) => mayRight.Contains(tokenid); + + + List> newTokens = new List>(); + for (int i = 0; i < tokens.Count; i++) + { + if ( i >= 1 && + mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID)) + { + newTokens.Add(new Token() + { + TokenID = FormulaToken.TIMES + }); + } + newTokens.Add(tokens[i]); + } + + return newTokens; + } + private static void TestLexerPostProcess() + { + var parserInstance = new VariableExpressionParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression", + lexerPostProcess: postProcess); + if (build.IsError) + { + foreach (var error in build.Errors) + { + Console.WriteLine(error.Message); + } + + return; + } + + var Parser = build.Result; + var r = Parser.Parse("2 * x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); + + + r = Parser.Parse("2 x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("2 ( x ) "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("x x "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); + + } + + private static void TestLexerPostProcessEBNF() + { + var parserInstance = new FormulaParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, $"{typeof(FormulaParser).Name}_expressions", + lexerPostProcess: postProcessFormula); + if (build.IsError) + { + foreach (var error in build.Errors) + { + Console.WriteLine(error.Message); + } + + return; + } + + var Parser = build.Result; + var r = Parser.Parse("2 * x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); + + + r = Parser.Parse("2 x"); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("2 ( x ) "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("x x "); + if (r.IsError) + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + + return; + } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() + { { "x", 2 } })); + Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); + + } } public enum TestGrammarToken diff --git a/samples/ParserExample/expressionModel/BinaryOperation.cs b/samples/ParserExample/expressionModel/BinaryOperation.cs new file mode 100644 index 00000000..7e084531 --- /dev/null +++ b/samples/ParserExample/expressionModel/BinaryOperation.cs @@ -0,0 +1,56 @@ +using System; +using expressionparser.model; + +namespace ParserExample.expressionModel +{ + public class BinaryOperation : Expression + { + private readonly Expression LeftExpresion; + private readonly FormulaToken Operator; + private readonly Expression RightExpression; + + + public BinaryOperation(Expression left, FormulaToken op, Expression right) + { + LeftExpresion = left; + Operator = op; + RightExpression = right; + } + + public double? Evaluate(ExpressionContext context) + { + var left = LeftExpresion.Evaluate(context); + var right = RightExpression.Evaluate(context); + + if (left.HasValue && right.HasValue) + switch (Operator) + { + case FormulaToken.PLUS: + { + return left.Value + right.Value; + } + case FormulaToken.MINUS: + { + return left.Value - right.Value; + } + case FormulaToken.TIMES: + { + return left.Value * right.Value; + } + case FormulaToken.DIVIDE: + { + return left.Value / right.Value; + } + case FormulaToken.EXP: + { + return Math.Pow(left.Value,right.Value); + } + default: + { + return null; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Expression.cs b/samples/ParserExample/expressionModel/Expression.cs new file mode 100644 index 00000000..26384cd8 --- /dev/null +++ b/samples/ParserExample/expressionModel/Expression.cs @@ -0,0 +1,9 @@ +using expressionparser.model; + +namespace ParserExample.expressionModel +{ + public interface Expression + { + double? Evaluate(ExpressionContext context); + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/ExpressionContext.cs b/samples/ParserExample/expressionModel/ExpressionContext.cs new file mode 100644 index 00000000..dfee22f4 --- /dev/null +++ b/samples/ParserExample/expressionModel/ExpressionContext.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; + +namespace ParserExample.expressionModel +{ + public class ExpressionContext + { + private readonly Dictionary Variables; + + public ExpressionContext() + { + Variables = new Dictionary(); + } + + public ExpressionContext(Dictionary variables) + { + Variables = variables; + } + + public double? GetValue(string variable) + { + if (Variables.ContainsKey(variable)) return Variables[variable]; + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FormulaParser.cs b/samples/ParserExample/expressionModel/FormulaParser.cs new file mode 100644 index 00000000..eef23c19 --- /dev/null +++ b/samples/ParserExample/expressionModel/FormulaParser.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserExample.expressionModel +{ + public class FormulaParser + { + [Operation((int) FormulaToken.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation((int) FormulaToken.MINUS, Affix.InFix, Associativity.Left, 10)] + public Expression BinaryTermExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + + [Operation((int) FormulaToken.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation((int) FormulaToken.DIVIDE, Affix.InFix, Associativity.Left, 50)] + public Expression BinaryFactorExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + + [Operation((int) FormulaToken.MINUS, Affix.PreFix, Associativity.Right, 100)] + public Expression PreFixExpression(Token operation, Expression value) + { + return new UnaryOperation(FormulaToken.MINUS,value); + } + + [Operation((int) FormulaToken.EXP, Affix.InFix, Associativity.Left, 90)] + public Expression ExpExpression(Expression left, Token operation, Expression right) + { + return new BinaryOperation(left, operation.TokenID, right); + } + + [Operand] + [Production("operand : primary_value")] + public Expression OperandValue(Expression value) + { + return value; + } + + + [Production("primary_value : IDENTIFIER")] + public Expression OperandVariable(Token identifier) + { + return new Variable(identifier.Value); + } + + [Production("primary_value : DOUBLE")] + [Production("primary_value : INT")] + public Expression OperandInt(Token value) + { + return new Number(value.DoubleValue); + } + + [Production("primary_value : LPAREN FormulaParser_expressions RPAREN")] + public Expression OperandParens(Token lparen, Expression value, Token rparen) + { + return value; + } + + [Production( + "primary_value : IDENTIFIER LPAREN[d] FormulaParser_expressions (COMMA FormulaParser_expressions)* RPAREN[d]")] + public Expression FunctionCall(Token funcName, Expression first, + List> others) + { + List parameters = new List(); + parameters.Add(first); + parameters.AddRange(others.Select(x => x.Value(0))); + return new FunctionCall(funcName.Value, parameters); + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FormulaToken.cs b/samples/ParserExample/expressionModel/FormulaToken.cs new file mode 100644 index 00000000..a5c396e3 --- /dev/null +++ b/samples/ParserExample/expressionModel/FormulaToken.cs @@ -0,0 +1,44 @@ +using sly.lexer; + +namespace ParserExample.expressionModel +{ + public enum FormulaToken + { + // float number + [Lexeme(GenericToken.Double)] DOUBLE = 1, + + // integer + [Lexeme(GenericToken.Int)] INT = 3, + + [Lexeme(GenericToken.Identifier)] IDENTIFIER = 4, + + // the + operator + [Lexeme(GenericToken.SugarToken, "+")] PLUS = 5, + + // the ++ operator + [Lexeme(GenericToken.SugarToken, "++")] + INCREMENT = 6, + + // the - operator + [Lexeme(GenericToken.SugarToken, "-")] MINUS = 7, + + // the * operator + [Lexeme(GenericToken.SugarToken, "*")] TIMES = 8, + + // the / operator + [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 9, + + // a left paranthesis ( + [Lexeme(GenericToken.SugarToken, "(")] LPAREN = 10, + + // a right paranthesis ) + [Lexeme(GenericToken.SugarToken, ")")] RPAREN = 11, + + [Sugar(",")]COMMA=12, + // a variable + + [Sugar("^")] EXP=13 + + + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/FunctionCall.cs b/samples/ParserExample/expressionModel/FunctionCall.cs new file mode 100644 index 00000000..38499553 --- /dev/null +++ b/samples/ParserExample/expressionModel/FunctionCall.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ParserExample.expressionModel +{ + public class FunctionCall : Expression + { + + public static double Sin(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Sin(parameters.First().Value); + } + + return 0.0; + } + + public static double Cos(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Cos(parameters.First().Value); + } + return 0.0; + } + + public static double Tan(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Tan(parameters.First().Value); + } + return 0.0; + } + + public static double Sqrt(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Sqrt(parameters.First().Value); + } + return 0.0; + } + + public static double Ln(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Log(parameters.First().Value); + } + return 0.0; + } + + public static double Log2(IEnumerable parameters) + { + if (parameters.Count() == 1 && parameters.First().HasValue) + { + return Math.Log2(parameters.First().Value); + } + return 0.0; + } + + public List Parameters { get; set; } + + public string Name { get; set; } + + public FunctionCall(string name, List parameters) + { + Name = name; + Parameters = parameters; + } + + public double? Evaluate(ExpressionContext context) + { + var parameters = Parameters.Select(x => x.Evaluate(context)); + switch (Name) + { + case "sin" : + return Sin(parameters); + case "cos" : + return Cos(parameters); + case "tan" : + return Tan(parameters); + case "sqrt" : + return Sqrt(parameters); + case "ln" : + return Ln(parameters); + case "log" : + return Log2(parameters); + } + + return 0.0; + } + + + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Group.cs b/samples/ParserExample/expressionModel/Group.cs new file mode 100644 index 00000000..758bfadc --- /dev/null +++ b/samples/ParserExample/expressionModel/Group.cs @@ -0,0 +1,17 @@ +namespace ParserExample.expressionModel +{ + public class Group : ParserExample.expressionModel.Expression + { + private readonly ParserExample.expressionModel.Expression InnerExpression; + + public Group(ParserExample.expressionModel.Expression expr) + { + InnerExpression = expr; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return InnerExpression.Evaluate(context); + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Number.cs b/samples/ParserExample/expressionModel/Number.cs new file mode 100644 index 00000000..e5ed797b --- /dev/null +++ b/samples/ParserExample/expressionModel/Number.cs @@ -0,0 +1,17 @@ +namespace ParserExample.expressionModel +{ + public sealed class Number : ParserExample.expressionModel.Expression + { + private readonly double Value; + + public Number(double value) + { + Value = value; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return Value; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/UnaryOperation.cs b/samples/ParserExample/expressionModel/UnaryOperation.cs new file mode 100644 index 00000000..9c6667af --- /dev/null +++ b/samples/ParserExample/expressionModel/UnaryOperation.cs @@ -0,0 +1,37 @@ +namespace ParserExample.expressionModel +{ + public class UnaryOperation : ParserExample.expressionModel.Expression + { + private readonly FormulaToken Operator; + private readonly ParserExample.expressionModel.Expression RightExpression; + + public UnaryOperation(FormulaToken op, ParserExample.expressionModel.Expression right) + { + Operator = op; + RightExpression = right; + } + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + var right = RightExpression.Evaluate(context); + + if (right.HasValue) + switch (Operator) + { + case FormulaToken.PLUS: + { + return +right.Value; + } + case FormulaToken.MINUS: + { + return -right.Value; + } + default: + { + return null; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/expressionModel/Variable.cs b/samples/ParserExample/expressionModel/Variable.cs new file mode 100644 index 00000000..c1c91348 --- /dev/null +++ b/samples/ParserExample/expressionModel/Variable.cs @@ -0,0 +1,18 @@ +namespace ParserExample.expressionModel +{ + public sealed class Variable : ParserExample.expressionModel.Expression + { + private readonly string VariableName; + + public Variable(string varName) + { + VariableName = varName; + } + + + public double? Evaluate(ParserExample.expressionModel.ExpressionContext context) + { + return context.GetValue(VariableName); + } + } +} \ No newline at end of file diff --git a/samples/jsonparser/JSONLexer.cs b/samples/jsonparser/JSONLexer.cs index ebf05d50..64784ecc 100644 --- a/samples/jsonparser/JSONLexer.cs +++ b/samples/jsonparser/JSONLexer.cs @@ -8,7 +8,8 @@ namespace jsonparser public class JSONLexer : ILexer { public string I18n { get; set; } - + public LexerPostProcess LexerPostProcess { get; set; } + public void AddDefinition(TokenDefinition tokenDefinition) { } diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs index c01e6b0a..7afa2e5d 100644 --- a/sly/lexer/GenericLexer.cs +++ b/sly/lexer/GenericLexer.cs @@ -79,6 +79,8 @@ public Config() public IEqualityComparer KeyWordComparer => KeyWordIgnoreCase ? StringComparer.OrdinalIgnoreCase : null; } + public LexerPostProcess LexerPostProcess { get; set; } + public string I18n { get; set; } public const string in_string = "in_string"; diff --git a/sly/lexer/ILexer.cs b/sly/lexer/ILexer.cs index fc1ba453..d5300e1a 100644 --- a/sly/lexer/ILexer.cs +++ b/sly/lexer/ILexer.cs @@ -1,5 +1,6 @@  using System; +using sly.lexer.fsm; namespace sly.lexer { @@ -11,5 +12,7 @@ public interface ILexer where T : struct LexerResult Tokenize(ReadOnlyMemory source); string I18n { get; set; } + + LexerPostProcess LexerPostProcess { get; set; } } } \ No newline at end of file diff --git a/sly/lexer/Lexer.cs b/sly/lexer/Lexer.cs index 0b0a32f6..915bc738 100644 --- a/sly/lexer/Lexer.cs +++ b/sly/lexer/Lexer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using sly.lexer.fsm; namespace sly.lexer { @@ -10,7 +11,8 @@ namespace sly.lexer public class Lexer : ILexer where T : struct { public string I18n { get; set; } - + public LexerPostProcess LexerPostProcess { get; set; } + private readonly IList> tokenDefinitions = new List>(); public void AddDefinition(TokenDefinition tokenDefinition) diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index 33865318..b8d1a8e0 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -85,14 +85,15 @@ public static BuildResult> BuildLexer(BuildExtension extensio public static BuildResult> BuildLexer(BuildResult> result, BuildExtension extensionBuilder = null, - string lang = null) where IN : struct + string lang = null, LexerPostProcess lexerPostProcess = null) where IN : struct { var attributes = GetLexemes(result,lang); if (!result.IsError) { result = Build(attributes, result, extensionBuilder,lang); + result.Result.LexerPostProcess = lexerPostProcess; } - + return result; } diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs index 959a6992..68d39e8d 100644 --- a/sly/lexer/fsm/FSMLexer.cs +++ b/sly/lexer/fsm/FSMLexer.cs @@ -20,6 +20,8 @@ public static T At(this ReadOnlyMemory memory, LexerPosition position) } public delegate void BuildExtension(IN token, LexemeAttribute lexem, GenericLexer lexer) where IN : struct; + + public delegate List> LexerPostProcess(List> tokens) where IN : struct; public class FSMLexer { diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index d8000f8f..6397bb57 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -22,7 +22,7 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) } public override BuildResult> BuildParser(object parserInstance, ParserType parserType, - string rootRule, BuildExtension extensionBuilder = null) + string rootRule, BuildExtension extensionBuilder = null, LexerPostProcess lexerPostProcess = null) { var ruleparser = new RuleParser(); var builder = new ParserBuilder>(I18n); @@ -66,7 +66,7 @@ public override BuildResult> BuildParser(object parserInstance, visitor = new EBNFSyntaxTreeVisitor(configuration, parserInstance); var parser = new Parser(I18n,syntaxParser, visitor); parser.Configuration = configuration; - var lexerResult = BuildLexer(extensionBuilder); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 8ae64de5..097a8a9c 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -53,7 +53,7 @@ public ParserBuilder() : this(null) /// the name of the root non terminal of the grammar /// public virtual BuildResult> BuildParser(object parserInstance, ParserType parserType, - string rootRule, BuildExtension extensionBuilder = null) + string rootRule, BuildExtension extensionBuilder = null, LexerPostProcess lexerPostProcess = null) { Parser parser = null; var result = new BuildResult>(); @@ -74,7 +74,7 @@ public virtual BuildResult> BuildParser(object parserInstance, P var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); var visitor = new SyntaxTreeVisitor(configuration, parserInstance); parser = new Parser(I18n,syntaxParser, visitor); - var lexerResult = BuildLexer(extensionBuilder); + var lexerResult = BuildLexer(extensionBuilder, lexerPostProcess); parser.Lexer = lexerResult.Result; if (lexerResult.IsError) { @@ -89,7 +89,7 @@ public virtual BuildResult> BuildParser(object parserInstance, P { var builder = new EBNFParserBuilder(I18n); result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule, - extensionBuilder); + extensionBuilder,lexerPostProcess); } parser = result.Result; @@ -161,9 +161,10 @@ private Tuple ExtractNTAndRule(string ruleString) } - protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null) + protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null, + LexerPostProcess lexerPostProcess = null) { - var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n); + var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess); return lexer; } diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs index 8d4436f9..21a331e0 100644 --- a/sly/parser/parser/Parser.cs +++ b/sly/parser/parser/Parser.cs @@ -74,6 +74,10 @@ public ParseResult ParseWithContext(string source, object context, stri } var tokens = lexingResult.Tokens; + if (Lexer.LexerPostProcess != null) + { + tokens = Lexer.LexerPostProcess(tokens); + } var position = 0; var tokensWithoutComments = new List>(); for (var i = 0; i < tokens.Count; i++) diff --git a/sly/sly.csproj b/sly/sly.csproj index c98d0261..39fd71e2 100644 --- a/sly/sly.csproj +++ b/sly/sly.csproj @@ -6,11 +6,11 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 2.7.0.3 + 2.7.0.4-alpha.2 https://github.com/b3b00/sly https://github.com/b3b00/sly https://github.com/b3b00/sly/blob/master/LICENSE - 2.7.0.3 + 2.7.0.4-alpha.2 Library