diff --git a/ParserTests/ParserTests.csproj b/ParserTests/ParserTests.csproj
index 04c0f430..e24f69c3 100644
--- a/ParserTests/ParserTests.csproj
+++ b/ParserTests/ParserTests.csproj
@@ -25,6 +25,7 @@
+
diff --git a/ParserTests/PostProcessedLexerTests.cs b/ParserTests/PostProcessedLexerTests.cs
new file mode 100644
index 00000000..c44cc54d
--- /dev/null
+++ b/ParserTests/PostProcessedLexerTests.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using Xunit;
+using postProcessedLexerParser;
+using postProcessedLexerParser.expressionModel;
+
+namespace ParserTests
+{
+ public class PostProcessedLexerTests
+ {
+ [Fact]
+ public void TestPostLexerProcessing()
+ {
+ var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser();
+
+ var r = Parser.Parse("2 * x");
+ Assert.False(r.IsError);
+ var res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Assert.NotNull(res);
+ Assert.Equal(4,res.Value);
+
+
+ r = Parser.Parse("2 x");
+ Assert.False(r.IsError);
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Assert.NotNull(res);
+ Assert.Equal(4,res.Value);
+
+
+ r = Parser.Parse("2 ( x ) ");
+ Assert.False(r.IsError);
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Assert.NotNull(res);
+ Assert.Equal(4,res.Value);
+
+ }
+ }
+}
\ No newline at end of file
diff --git a/ParserTests/comments/CommentsErrorTest.cs b/ParserTests/comments/CommentsErrorTest.cs
index a967ba0b..039f566f 100644
--- a/ParserTests/comments/CommentsErrorTest.cs
+++ b/ParserTests/comments/CommentsErrorTest.cs
@@ -184,7 +184,7 @@ public void RedundantAttributes()
[Fact]
public void MixedErrors()
{
- var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en");
+ var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en");
Assert.True(lexerRes10.IsError);
Assert.Equal(4, lexerRes10.Errors.Count);
var expectedErrors = new[]
@@ -199,7 +199,7 @@ public void MixedErrors()
expectedErrors = new[] {"too many multi-line comment lexem", "too many single-line comment lexem"};
- var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en");
+ var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en");
Assert.True(lexerRes9.IsError);
Assert.Equal(2, lexerRes9.Errors.Count);
foreach (var expectedError in expectedErrors)
@@ -209,7 +209,7 @@ public void MixedErrors()
expectedErrors = new[] {"too many multi-line comment lexem","comment lexem can't be used together with single-line or multi-line comment lexems"};
- var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult>(),lang:"en");
+ var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult>(),lang: "en");
Assert.True(lexerRes8.IsError);
Assert.Equal(2, lexerRes8.Errors.Count);
foreach (var expectedError in expectedErrors)
diff --git a/samples/Issue244/Issue244Parser.cs b/samples/Issue244/Issue244Parser.cs
new file mode 100644
index 00000000..a55f57b2
--- /dev/null
+++ b/samples/Issue244/Issue244Parser.cs
@@ -0,0 +1,152 @@
+using System;
+using expressionparser;
+using simpleExpressionParser;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace Issue244
+{
+ public class Issue244Parser
+ {
+ [Operation((int) SimpleExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)]
+ [Operation("MINUS", Affix.InFix, Associativity.Left, 10)]
+ public Result BinaryTermExpression(Result left, Token operation, Result right)
+ {
+ // Get the length of the expression :
+ int length = right.EndIndex - left.StartIndex;
+
+
+ Result result = null;
+ switch (operation.TokenID)
+ {
+ case SimpleExpressionToken.PLUS:
+ {
+ result = new Result()
+ {
+ Value = left.Value + right.Value,
+ StartColumn = left.StartColumn,
+ EndColumn = right.EndColumn,
+ StartIndex = left.StartIndex,
+ EndIndex = right.EndIndex
+ }
+ ;
+ break;
+ }
+ case SimpleExpressionToken.MINUS:
+ {
+ result = new Result()
+ {
+ Value = left.Value - right.Value,
+ StartColumn = left.StartColumn,
+ EndColumn = right.EndColumn,
+ StartIndex = left.StartIndex,
+ EndIndex = right.EndIndex
+ };
+ break;
+ }
+ }
+
+ return result;
+ }
+
+
+ [Operation((int) SimpleExpressionToken.TIMES, Affix.InFix, Associativity.Right, 50)]
+ [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)]
+ public Result BinaryFactorExpression(Result left, Token operation, Result right)
+ {
+ Result result = null;
+ switch (operation.TokenID)
+ {
+ case SimpleExpressionToken.TIMES:
+ {
+ result = new Result()
+ {
+ Value = left.Value * right.Value,
+ StartColumn = left.StartColumn,
+ EndColumn = right.EndColumn,
+ StartIndex = left.StartIndex,
+ EndIndex = right.EndIndex
+ };
+ break;
+ }
+ case SimpleExpressionToken.DIVIDE:
+ {
+ result = new Result()
+ {
+ Value = left.Value / right.Value,
+ StartColumn = left.StartColumn,
+ EndColumn = right.EndColumn,
+ StartIndex = left.StartIndex,
+ EndIndex = right.EndIndex
+ };
+ break;
+ }
+ }
+
+ return result;
+ }
+
+
+ [Operation((int) SimpleExpressionToken.MINUS, Affix.PreFix, Associativity.Right, 100)]
+ public Result PreFixExpression(Token operation, Result value)
+ {
+ return new Result()
+ {
+ Value = -value.Value,
+ StartColumn = operation.Position.Column,
+ EndColumn = value.EndColumn,
+ StartIndex = operation.Position.Index,
+ EndIndex = value.EndIndex
+ };
+ }
+
+
+
+ [Operand]
+ [Production("operand : primary_value")]
+ public Result OperandValue(Result value)
+ {
+ return value;
+ }
+
+
+ [Production("primary_value : DOUBLE")]
+ public Result OperandDouble(Token value)
+ {
+ return new Result()
+ {
+ Value = value.DoubleValue,
+ StartColumn = value.Position.Column,
+ EndColumn = value.Position.Column + value.Value.Length,
+ StartIndex = value.Position.Index,
+ EndIndex = value.Position.Index + value.Value.Length
+ };
+ }
+
+ [Production("primary_value : INT")]
+ public Result OperandInt(Token value)
+ {
+ return new Result()
+ {
+ Value = value.DoubleValue,
+ StartColumn = value.Position.Column,
+ EndColumn = value.Position.Column + value.Value.Length,
+ StartIndex = value.Position.Index,
+ EndIndex = value.Position.Index + value.Value.Length
+ };
+ }
+
+ [Production("primary_value : LPAREN Issue244Parser_expressions RPAREN")]
+ public Result OperandParens(Token lparen, Result value, Token rparen)
+ {
+ return new Result()
+ {
+ Value = value.Value,
+ StartColumn = lparen.Position.Column,
+ EndColumn = rparen.Position.Column + rparen.Value.Length,
+ StartIndex = rparen.Position.Index,
+ EndIndex = rparen.Position.Index + rparen.Value.Length,
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj
index e40db501..a5b10c33 100644
--- a/samples/ParserExample/ParserExample.csproj
+++ b/samples/ParserExample/ParserExample.csproj
@@ -30,6 +30,7 @@
+
diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs
index 41e631c4..1808f777 100644
--- a/samples/ParserExample/Program.cs
+++ b/samples/ParserExample/Program.cs
@@ -18,6 +18,7 @@
using ParserTests;
using ParserTests.Issue239;
using ParserTests.lexer;
+using postProcessedLexerParser.expressionModel;
using simpleExpressionParser;
using sly.lexer;
using sly.lexer.fsm;
@@ -29,6 +30,8 @@
using sly.parser.generator.visitor;
using sly.parser.parser;
using Xunit;
+using Expression = postProcessedLexerParser.expressionModel.Expression;
+using ExpressionContext = postProcessedLexerParser.expressionModel.ExpressionContext;
namespace ParserExample
{
@@ -928,8 +931,12 @@ private static void Main(string[] args)
// TestI18N();
// Console.ReadLine();
// TestShortGeneric();
+
+ //TestIssue239();
+ TestLexerPostProcess();
+ TestLexerPostProcessEBNF();
//TestIssue239();
- TestShortOperations();
+ // TestShortOperations();
}
@@ -986,6 +993,169 @@ private static void TestIssue239()
Issue239Tests.TestOk();
;
}
+
+ private static List> postProcess(List> tokens)
+ {
+ var mayLeft = new List()
+ {
+ ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.IDENTIFIER
+ };
+
+ var mayRight = new List()
+ {
+ ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.LPAREN, ExpressionToken.IDENTIFIER
+ };
+
+ Func mayOmmitLeft = (ExpressionToken tokenid) => mayLeft.Contains(tokenid);
+
+ Func mayOmmitRight = (ExpressionToken tokenid) => mayRight.Contains(tokenid);
+
+
+ List> newTokens = new List>();
+ for (int i = 0; i < tokens.Count; i++)
+ {
+ if ( i >= 1 &&
+ mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID))
+ {
+ newTokens.Add(new Token()
+ {
+ TokenID = ExpressionToken.TIMES
+ });
+ }
+ newTokens.Add(tokens[i]);
+ }
+
+ return newTokens;
+ }
+
+
+
+ private static void TestLexerPostProcess()
+ {
+ var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser();
+
+ var r = Parser.Parse("2 * x");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ var res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+
+ r = Parser.Parse("2 x");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ r = Parser.Parse("2 ( x ) ");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ r = Parser.Parse("x x ");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ }
+
+ private static void TestLexerPostProcessEBNF()
+ {
+ var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser();
+
+ var r = Parser.Parse("2 * x");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ var res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+
+ r = Parser.Parse("2 x");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ r = Parser.Parse("2 ( x ) ");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+ Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ r = Parser.Parse("x x ");
+ if (r.IsError)
+ {
+ foreach (var error in r.Errors)
+ {
+ Console.WriteLine(error.ErrorMessage);
+ }
+
+ return;
+ }
+ res = r.Result.Evaluate(new ExpressionContext(new Dictionary()
+ { { "x", 2 } }));
+
+ Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?"));
+
+ }
}
public enum TestGrammarToken
diff --git a/samples/jsonparser/JSONLexer.cs b/samples/jsonparser/JSONLexer.cs
index ebf05d50..64784ecc 100644
--- a/samples/jsonparser/JSONLexer.cs
+++ b/samples/jsonparser/JSONLexer.cs
@@ -8,7 +8,8 @@ namespace jsonparser
public class JSONLexer : ILexer
{
public string I18n { get; set; }
-
+ public LexerPostProcess LexerPostProcess { get; set; }
+
public void AddDefinition(TokenDefinition tokenDefinition)
{
}
diff --git a/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs b/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs
new file mode 100644
index 00000000..2f251efc
--- /dev/null
+++ b/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs
@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using postProcessedLexerParser.expressionModel;
+using sly.lexer;
+using sly.parser;
+using sly.parser.generator;
+
+namespace postProcessedLexerParser
+{
+ public class PostProcessedLexerParserBuilder
+ {
+
+ private static List> postProcessFormula(List> tokens)
+ {
+ var mayLeft = new List()
+ {
+ FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.IDENTIFIER
+ };
+
+ var mayRight = new List()
+ {
+ FormulaToken.INT, FormulaToken.DOUBLE, FormulaToken.LPAREN, FormulaToken.IDENTIFIER
+ };
+
+ Func mayOmmitLeft = (FormulaToken tokenid) => mayLeft.Contains(tokenid);
+
+ Func mayOmmitRight = (FormulaToken tokenid) => mayRight.Contains(tokenid);
+
+
+ List> newTokens = new List>();
+ for (int i = 0; i < tokens.Count; i++)
+ {
+ if ( i >= 1 &&
+ mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID))
+ {
+ newTokens.Add(new Token()
+ {
+ TokenID = FormulaToken.TIMES
+ });
+ }
+ newTokens.Add(tokens[i]);
+ }
+
+ return newTokens;
+ }
+
+ public static Parser buildPostProcessedLexerParser()
+ {
+ var parserInstance = new FormulaParser();
+ var builder = new ParserBuilder();
+ var build = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, $"{typeof(FormulaParser).Name}_expressions",
+ lexerPostProcess: postProcessFormula);
+ if (build.IsError)
+ {
+ foreach (var error in build.Errors)
+ {
+ Console.WriteLine(error.Message);
+ }
+
+ return null;
+ }
+
+ var Parser = build.Result;
+ return Parser;
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/BinaryOperation.cs b/samples/postProcessedLexerParser/expressionModel/BinaryOperation.cs
new file mode 100644
index 00000000..a99f8c19
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/BinaryOperation.cs
@@ -0,0 +1,55 @@
+using System;
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public class BinaryOperation : Expression
+ {
+ private readonly Expression LeftExpresion;
+ private readonly FormulaToken Operator;
+ private readonly Expression RightExpression;
+
+
+ public BinaryOperation(Expression left, FormulaToken op, Expression right)
+ {
+ LeftExpresion = left;
+ Operator = op;
+ RightExpression = right;
+ }
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ var left = LeftExpresion.Evaluate(context);
+ var right = RightExpression.Evaluate(context);
+
+ if (left.HasValue && right.HasValue)
+ switch (Operator)
+ {
+ case FormulaToken.PLUS:
+ {
+ return left.Value + right.Value;
+ }
+ case FormulaToken.MINUS:
+ {
+ return left.Value - right.Value;
+ }
+ case FormulaToken.TIMES:
+ {
+ return left.Value * right.Value;
+ }
+ case FormulaToken.DIVIDE:
+ {
+ return left.Value / right.Value;
+ }
+ case FormulaToken.EXP:
+ {
+ return Math.Pow(left.Value,right.Value);
+ }
+ default:
+ {
+ return null;
+ }
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/Expression.cs b/samples/postProcessedLexerParser/expressionModel/Expression.cs
new file mode 100644
index 00000000..791228bf
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/Expression.cs
@@ -0,0 +1,8 @@
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public interface Expression
+ {
+ double? Evaluate(ExpressionContext context);
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/ExpressionContext.cs b/samples/postProcessedLexerParser/expressionModel/ExpressionContext.cs
new file mode 100644
index 00000000..4cbf7063
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/ExpressionContext.cs
@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public class ExpressionContext
+ {
+ private readonly Dictionary Variables;
+
+ public ExpressionContext()
+ {
+ Variables = new Dictionary();
+ }
+
+ public ExpressionContext(Dictionary variables)
+ {
+ Variables = variables;
+ }
+
+ public double? GetValue(string variable)
+ {
+ if (Variables.ContainsKey(variable)) return Variables[variable];
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/FormulaParser.cs b/samples/postProcessedLexerParser/expressionModel/FormulaParser.cs
new file mode 100644
index 00000000..a7ecb11e
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/FormulaParser.cs
@@ -0,0 +1,78 @@
+using System.Collections.Generic;
+using System.Linq;
+using postProcessedLexerParser.expressionModel;
+using sly.lexer;
+using sly.parser.generator;
+using sly.parser.parser;
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public class FormulaParser
+ {
+ [Operation((int) FormulaToken.PLUS, Affix.InFix, Associativity.Right, 10)]
+ [Operation((int) FormulaToken.MINUS, Affix.InFix, Associativity.Left, 10)]
+ public Expression BinaryTermExpression(Expression left, Token operation, Expression right)
+ {
+ return new BinaryOperation(left, operation.TokenID, right);
+ }
+
+
+ [Operation((int) FormulaToken.TIMES, Affix.InFix, Associativity.Right, 50)]
+ [Operation((int) FormulaToken.DIVIDE, Affix.InFix, Associativity.Left, 50)]
+ public Expression BinaryFactorExpression(Expression left, Token operation, Expression right)
+ {
+ return new BinaryOperation(left, operation.TokenID, right);
+ }
+
+
+ [Operation((int) FormulaToken.MINUS, Affix.PreFix, Associativity.Right, 100)]
+ public Expression PreFixExpression(Token operation, Expression value)
+ {
+ return new UnaryOperation(FormulaToken.MINUS,value);
+ }
+
+ [Operation((int) FormulaToken.EXP, Affix.InFix, Associativity.Left, 90)]
+ public Expression ExpExpression(Expression left, Token operation, Expression right)
+ {
+ return new BinaryOperation(left, operation.TokenID, right);
+ }
+
+ [Operand]
+ [Production("operand : primary_value")]
+ public Expression OperandValue(Expression value)
+ {
+ return value;
+ }
+
+
+ [Production("primary_value : IDENTIFIER")]
+ public Expression OperandVariable(Token identifier)
+ {
+ return new Variable(identifier.Value);
+ }
+
+ [Production("primary_value : DOUBLE")]
+ [Production("primary_value : INT")]
+ public Expression OperandInt(Token value)
+ {
+ return new Number(value.DoubleValue);
+ }
+
+ [Production("primary_value : LPAREN FormulaParser_expressions RPAREN")]
+ public Expression OperandParens(Token lparen, Expression value, Token rparen)
+ {
+ return value;
+ }
+
+ [Production(
+ "primary_value : IDENTIFIER LPAREN[d] FormulaParser_expressions (COMMA FormulaParser_expressions)* RPAREN[d]")]
+ public Expression FunctionCall(Token funcName, Expression first,
+ List> others)
+ {
+ List parameters = new List();
+ parameters.Add(first);
+ parameters.AddRange(others.Select(x => x.Value(0)));
+ return new FunctionCall(funcName.Value, parameters);
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/FormulaToken.cs b/samples/postProcessedLexerParser/expressionModel/FormulaToken.cs
new file mode 100644
index 00000000..88bd8f9f
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/FormulaToken.cs
@@ -0,0 +1,44 @@
+using sly.lexer;
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public enum FormulaToken
+ {
+ // float number
+ [Lexeme(GenericToken.Double)] DOUBLE = 1,
+
+ // integer
+ [Lexeme(GenericToken.Int)] INT = 3,
+
+ [Lexeme(GenericToken.Identifier)] IDENTIFIER = 4,
+
+ // the + operator
+ [Lexeme(GenericToken.SugarToken, "+")] PLUS = 5,
+
+ // the ++ operator
+ [Lexeme(GenericToken.SugarToken, "++")]
+ INCREMENT = 6,
+
+ // the - operator
+ [Lexeme(GenericToken.SugarToken, "-")] MINUS = 7,
+
+ // the * operator
+ [Lexeme(GenericToken.SugarToken, "*")] TIMES = 8,
+
+ // the / operator
+ [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 9,
+
+ // a left paranthesis (
+ [Lexeme(GenericToken.SugarToken, "(")] LPAREN = 10,
+
+ // a right paranthesis )
+ [Lexeme(GenericToken.SugarToken, ")")] RPAREN = 11,
+
+ [Sugar(",")]COMMA=12,
+ // a variable
+
+ [Sugar("^")] EXP=13
+
+
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/FunctionCall.cs b/samples/postProcessedLexerParser/expressionModel/FunctionCall.cs
new file mode 100644
index 00000000..efa63735
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/FunctionCall.cs
@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace postProcessedLexerParser.expressionModel
+{
+ public class FunctionCall : Expression
+ {
+
+ public static double Sin(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Sin(parameters.First().Value);
+ }
+
+ return 0.0;
+ }
+
+ public static double Cos(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Cos(parameters.First().Value);
+ }
+ return 0.0;
+ }
+
+ public static double Tan(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Tan(parameters.First().Value);
+ }
+ return 0.0;
+ }
+
+ public static double Sqrt(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Sqrt(parameters.First().Value);
+ }
+ return 0.0;
+ }
+
+ public static double Ln(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Log(parameters.First().Value);
+ }
+ return 0.0;
+ }
+
+ public static double Log2(IEnumerable parameters)
+ {
+ if (parameters.Count() == 1 && parameters.First().HasValue)
+ {
+ return Math.Log(parameters.First().Value);
+ }
+ return 0.0;
+ }
+
+ public List Parameters { get; set; }
+
+ public string Name { get; set; }
+
+ public FunctionCall(string name, List parameters)
+ {
+ Name = name;
+ Parameters = parameters;
+ }
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ var parameters = Parameters.Select(x => x.Evaluate(context));
+ switch (Name)
+ {
+ case "sin" :
+ return Sin(parameters);
+ case "cos" :
+ return Cos(parameters);
+ case "tan" :
+ return Tan(parameters);
+ case "sqrt" :
+ return Sqrt(parameters);
+ case "ln" :
+ return Ln(parameters);
+ case "log" :
+ return Log2(parameters);
+ }
+
+ return 0.0;
+ }
+
+
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/Group.cs b/samples/postProcessedLexerParser/expressionModel/Group.cs
new file mode 100644
index 00000000..ffdd797e
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/Group.cs
@@ -0,0 +1,17 @@
+namespace postProcessedLexerParser.expressionModel
+{
+ public class Group : Expression
+ {
+ private readonly Expression InnerExpression;
+
+ public Group(Expression expr)
+ {
+ InnerExpression = expr;
+ }
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ return InnerExpression.Evaluate(context);
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/Number.cs b/samples/postProcessedLexerParser/expressionModel/Number.cs
new file mode 100644
index 00000000..6214871e
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/Number.cs
@@ -0,0 +1,17 @@
+namespace postProcessedLexerParser.expressionModel
+{
+ public sealed class Number : Expression
+ {
+ private readonly double Value;
+
+ public Number(double value)
+ {
+ Value = value;
+ }
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ return Value;
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/UnaryOperation.cs b/samples/postProcessedLexerParser/expressionModel/UnaryOperation.cs
new file mode 100644
index 00000000..35bbe35f
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/UnaryOperation.cs
@@ -0,0 +1,37 @@
+namespace postProcessedLexerParser.expressionModel
+{
+ public class UnaryOperation : Expression
+ {
+ private readonly FormulaToken Operator;
+ private readonly Expression RightExpression;
+
+ public UnaryOperation(FormulaToken op, Expression right)
+ {
+ Operator = op;
+ RightExpression = right;
+ }
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ var right = RightExpression.Evaluate(context);
+
+ if (right.HasValue)
+ switch (Operator)
+ {
+ case FormulaToken.PLUS:
+ {
+ return +right.Value;
+ }
+ case FormulaToken.MINUS:
+ {
+ return -right.Value;
+ }
+ default:
+ {
+ return null;
+ }
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/expressionModel/Variable.cs b/samples/postProcessedLexerParser/expressionModel/Variable.cs
new file mode 100644
index 00000000..c5ec0763
--- /dev/null
+++ b/samples/postProcessedLexerParser/expressionModel/Variable.cs
@@ -0,0 +1,18 @@
+namespace postProcessedLexerParser.expressionModel
+{
+ public sealed class Variable : Expression
+ {
+ private readonly string VariableName;
+
+ public Variable(string varName)
+ {
+ VariableName = varName;
+ }
+
+
+ public double? Evaluate(ExpressionContext context)
+ {
+ return context.GetValue(VariableName);
+ }
+ }
+}
\ No newline at end of file
diff --git a/samples/postProcessedLexerParser/postProcessedLexerParser.csproj b/samples/postProcessedLexerParser/postProcessedLexerParser.csproj
new file mode 100644
index 00000000..5dc8a2c6
--- /dev/null
+++ b/samples/postProcessedLexerParser/postProcessedLexerParser.csproj
@@ -0,0 +1,11 @@
+
+
+
+ netstandard2.1
+
+
+
+
+
+
+
diff --git a/sly.sln b/sly.sln
index 7755d7b6..cb988926 100644
--- a/sly.sln
+++ b/sly.sln
@@ -33,6 +33,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "indented", "samples\indente
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "indentedWhile", "samples\IndentedWhile\indentedWhile.csproj", "{B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "postProcessedLexerParser", "samples\postProcessedLexerParser\postProcessedLexerParser.csproj", "{8F703C3C-0D10-4F43-AC2E-14843D59E460}"
+EndProject
Global
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
@@ -102,6 +104,10 @@ Global
{B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4}.Release|Any CPU.Build.0 = Release|Any CPU
+ {8F703C3C-0D10-4F43-AC2E-14843D59E460}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {8F703C3C-0D10-4F43-AC2E-14843D59E460}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {8F703C3C-0D10-4F43-AC2E-14843D59E460}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {8F703C3C-0D10-4F43-AC2E-14843D59E460}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -115,6 +121,7 @@ Global
{FFCA5A16-0388-4A8E-8B40-0D1FDA5D4B7B} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
{7C6C4A13-2AE5-4070-A2E1-05B6A83DC97C} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
{B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
+ {8F703C3C-0D10-4F43-AC2E-14843D59E460} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {43254130-CF3E-480E-952F-E50CA5D2E417}
diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs
index c01e6b0a..7afa2e5d 100644
--- a/sly/lexer/GenericLexer.cs
+++ b/sly/lexer/GenericLexer.cs
@@ -79,6 +79,8 @@ public Config()
public IEqualityComparer KeyWordComparer => KeyWordIgnoreCase ? StringComparer.OrdinalIgnoreCase : null;
}
+ public LexerPostProcess LexerPostProcess { get; set; }
+
public string I18n { get; set; }
public const string in_string = "in_string";
diff --git a/sly/lexer/ILexer.cs b/sly/lexer/ILexer.cs
index fc1ba453..d5300e1a 100644
--- a/sly/lexer/ILexer.cs
+++ b/sly/lexer/ILexer.cs
@@ -1,5 +1,6 @@
using System;
+using sly.lexer.fsm;
namespace sly.lexer
{
@@ -11,5 +12,7 @@ public interface ILexer where T : struct
LexerResult Tokenize(ReadOnlyMemory source);
string I18n { get; set; }
+
+ LexerPostProcess LexerPostProcess { get; set; }
}
}
\ No newline at end of file
diff --git a/sly/lexer/Lexer.cs b/sly/lexer/Lexer.cs
index 0b0a32f6..915bc738 100644
--- a/sly/lexer/Lexer.cs
+++ b/sly/lexer/Lexer.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using sly.lexer.fsm;
namespace sly.lexer
{
@@ -10,7 +11,8 @@ namespace sly.lexer
public class Lexer : ILexer where T : struct
{
public string I18n { get; set; }
-
+ public LexerPostProcess LexerPostProcess { get; set; }
+
private readonly IList> tokenDefinitions = new List>();
public void AddDefinition(TokenDefinition tokenDefinition)
diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs
index 33865318..b8d1a8e0 100644
--- a/sly/lexer/LexerBuilder.cs
+++ b/sly/lexer/LexerBuilder.cs
@@ -85,14 +85,15 @@ public static BuildResult> BuildLexer(BuildExtension extensio
public static BuildResult> BuildLexer(BuildResult> result,
BuildExtension extensionBuilder = null,
- string lang = null) where IN : struct
+ string lang = null, LexerPostProcess lexerPostProcess = null) where IN : struct
{
var attributes = GetLexemes(result,lang);
if (!result.IsError)
{
result = Build(attributes, result, extensionBuilder,lang);
+ result.Result.LexerPostProcess = lexerPostProcess;
}
-
+
return result;
}
diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs
index 8d78ff68..6d44323f 100644
--- a/sly/lexer/fsm/FSMLexer.cs
+++ b/sly/lexer/fsm/FSMLexer.cs
@@ -20,6 +20,8 @@ public static T At(this ReadOnlyMemory memory, LexerPosition position)
}
public delegate void BuildExtension(IN token, LexemeAttribute lexem, GenericLexer lexer) where IN : struct;
+
+ public delegate List> LexerPostProcess(List> tokens) where IN : struct;
public class FSMLexer
{
diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs
index d8000f8f..6397bb57 100644
--- a/sly/parser/generator/EBNFParserBuilder.cs
+++ b/sly/parser/generator/EBNFParserBuilder.cs
@@ -22,7 +22,7 @@ public EBNFParserBuilder(string i18n = null) : base(i18n)
}
public override BuildResult> BuildParser(object parserInstance, ParserType parserType,
- string rootRule, BuildExtension extensionBuilder = null)
+ string rootRule, BuildExtension extensionBuilder = null, LexerPostProcess lexerPostProcess = null)
{
var ruleparser = new RuleParser();
var builder = new ParserBuilder>(I18n);
@@ -66,7 +66,7 @@ public override BuildResult> BuildParser(object parserInstance,
visitor = new EBNFSyntaxTreeVisitor(configuration, parserInstance);
var parser = new Parser(I18n,syntaxParser, visitor);
parser.Configuration = configuration;
- var lexerResult = BuildLexer(extensionBuilder);
+ var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess);
if (lexerResult.IsError)
{
foreach (var lexerResultError in lexerResult.Errors)
diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs
index 8ae64de5..097a8a9c 100644
--- a/sly/parser/generator/ParserBuilder.cs
+++ b/sly/parser/generator/ParserBuilder.cs
@@ -53,7 +53,7 @@ public ParserBuilder() : this(null)
/// the name of the root non terminal of the grammar
///
public virtual BuildResult> BuildParser(object parserInstance, ParserType parserType,
- string rootRule, BuildExtension extensionBuilder = null)
+ string rootRule, BuildExtension extensionBuilder = null, LexerPostProcess lexerPostProcess = null)
{
Parser parser = null;
var result = new BuildResult>();
@@ -74,7 +74,7 @@ public virtual BuildResult> BuildParser(object parserInstance, P
var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule);
var visitor = new SyntaxTreeVisitor(configuration, parserInstance);
parser = new Parser(I18n,syntaxParser, visitor);
- var lexerResult = BuildLexer(extensionBuilder);
+ var lexerResult = BuildLexer(extensionBuilder, lexerPostProcess);
parser.Lexer = lexerResult.Result;
if (lexerResult.IsError)
{
@@ -89,7 +89,7 @@ public virtual BuildResult> BuildParser(object parserInstance, P
{
var builder = new EBNFParserBuilder(I18n);
result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule,
- extensionBuilder);
+ extensionBuilder,lexerPostProcess);
}
parser = result.Result;
@@ -161,9 +161,10 @@ private Tuple ExtractNTAndRule(string ruleString)
}
- protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null)
+ protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null,
+ LexerPostProcess lexerPostProcess = null)
{
- var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n);
+ var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess);
return lexer;
}
diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs
index 380774e3..363a3bb4 100644
--- a/sly/parser/parser/Parser.cs
+++ b/sly/parser/parser/Parser.cs
@@ -74,6 +74,10 @@ public ParseResult ParseWithContext(string source, object context, stri
}
var tokens = lexingResult.Tokens;
+ if (Lexer.LexerPostProcess != null)
+ {
+ tokens = Lexer.LexerPostProcess(tokens);
+ }
var position = 0;
var tokensWithoutComments = new List>();
for (var i = 0; i < tokens.Count; i++)
diff --git a/sly/sly.csproj b/sly/sly.csproj
index 655e55a2..e6b75e86 100644
--- a/sly/sly.csproj
+++ b/sly/sly.csproj
@@ -6,11 +6,11 @@
$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb
#LY is a parser generator halfway between parser combinators and parser generator like ANTLR
b3b00
- 2.8.0.3
+ 2.8.0.4-alpha
https://github.com/b3b00/sly
https://github.com/b3b00/sly
https://github.com/b3b00/sly/blob/master/LICENSE
- 2.8.0.3
+ 2.8.0.4-alpha
Library