Merge pull request #266 from b3b00/experiment/lexer-post-processing

Experiment/lexer post processing
b3b00 · Jan 26, 2022 · e1e4450 · e1e4450
2 parents 815db22 + 7dcea54
commit e1e4450
Show file tree

Hide file tree

Showing 29 changed files with 881 additions and 17 deletions.
diff --git a/ParserTests/ParserTests.csproj b/ParserTests/ParserTests.csproj
@@ -25,6 +25,7 @@
     <ProjectReference Include="..\samples\GenericLexerWithCallbacks\GenericLexerWithCallbacks.csproj" />
     <ProjectReference Include="..\samples\IndentedWhile\indentedWhile.csproj" />
     <ProjectReference Include="..\samples\indented\indented.csproj" />
+    <ProjectReference Include="..\samples\postProcessedLexerParser\postProcessedLexerParser.csproj" />
     <ProjectReference Include="..\samples\SimpleExpressionParser\SimpleExpressionParser.csproj" />
     <ProjectReference Include="..\samples\while\while.csproj" />
     <ProjectReference Include="..\sly\sly.csproj" />

diff --git a/ParserTests/PostProcessedLexerTests.cs b/ParserTests/PostProcessedLexerTests.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using Xunit;
+using postProcessedLexerParser;
+using postProcessedLexerParser.expressionModel;
+
+namespace ParserTests
+{
+    public class PostProcessedLexerTests
+    {
+        [Fact]
+        public void TestPostLexerProcessing()
+        {
+            var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser();
+
+            var r = Parser.Parse("2 * x");
+            Assert.False(r.IsError);
+            var res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, double>()
+                { { "x", 2 } }));
+          Assert.NotNull(res);
+          Assert.Equal(4,res.Value);
+
+
+            r = Parser.Parse("2  x");
+            Assert.False(r.IsError);
+            res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, double>()
+                { { "x", 2 } }));
+            Assert.NotNull(res);
+            Assert.Equal(4,res.Value);
+
+
+            r = Parser.Parse("2 ( x ) ");
+            Assert.False(r.IsError);
+            res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, double>()
+                { { "x", 2 } }));
+            Assert.NotNull(res);
+            Assert.Equal(4,res.Value);
+
+        }
+    }
+}
diff --git a/ParserTests/comments/CommentsErrorTest.cs b/ParserTests/comments/CommentsErrorTest.cs
@@ -184,7 +184,7 @@ public void RedundantAttributes()
         [Fact]
         public void MixedErrors()
         {
-            var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError10>>(),lang:"en");
+            var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError10>>(),lang: "en");
             Assert.True(lexerRes10.IsError);
             Assert.Equal(4, lexerRes10.Errors.Count);
             var expectedErrors = new[]
@@ -199,7 +199,7 @@ public void MixedErrors()
 
             expectedErrors = new[] {"too many multi-line comment lexem", "too many single-line comment lexem"};
 
-            var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError9>>(),lang:"en");
+            var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError9>>(),lang: "en");
             Assert.True(lexerRes9.IsError);
             Assert.Equal(2, lexerRes9.Errors.Count);
             foreach (var expectedError in expectedErrors)
@@ -209,7 +209,7 @@ public void MixedErrors()
 
             expectedErrors = new[] {"too many multi-line comment lexem","comment lexem can't be used together with single-line or multi-line comment lexems"};
 
-            var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError8>>(),lang:"en");
+            var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError8>>(),lang: "en");
             Assert.True(lexerRes8.IsError);
             Assert.Equal(2, lexerRes8.Errors.Count);
             foreach (var expectedError in expectedErrors)

diff --git a/samples/Issue244/Issue244Parser.cs b/samples/Issue244/Issue244Parser.cs
@@ -0,0 +1,152 @@
+using System;
+using expressionparser;
+using simpleExpressionParser;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace Issue244
+{
+ public class Issue244Parser
+    {
+        [Operation((int) SimpleExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)]
+        [Operation("MINUS", Affix.InFix, Associativity.Left, 10)]
+        public Result BinaryTermExpression(Result left, Token<SimpleExpressionToken> operation, Result right)
+        {
+            // Get the length of the expression :
+            int length = right.EndIndex - left.StartIndex;
+
+
+            Result result = null;
+            switch (operation.TokenID)
+            {
+                case SimpleExpressionToken.PLUS:
+                {
+                    result = new Result()
+                    {
+                        Value = left.Value + right.Value, 
+                        StartColumn = left.StartColumn,
+                        EndColumn = right.EndColumn,
+                        StartIndex = left.StartIndex,
+                        EndIndex = right.EndIndex
+                    }
+                    ;
+                    break;
+                }
+                case SimpleExpressionToken.MINUS:
+                {
+                    result = new Result()
+                    {
+                        Value = left.Value - right.Value,
+                        StartColumn = left.StartColumn,
+                        EndColumn = right.EndColumn,
+                        StartIndex = left.StartIndex,
+                        EndIndex = right.EndIndex
+                    };
+                    break;
+                }
+            }
+
+            return result;
+        }
+
+
+        [Operation((int) SimpleExpressionToken.TIMES, Affix.InFix, Associativity.Right, 50)]
+        [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)]
+        public Result BinaryFactorExpression(Result left, Token<SimpleExpressionToken> operation, Result right)
+        {
+            Result result = null;
+            switch (operation.TokenID)
+            {
+                case SimpleExpressionToken.TIMES:
+                {
+                    result = new Result()
+                    {
+                        Value = left.Value * right.Value,
+                        StartColumn = left.StartColumn,
+                        EndColumn = right.EndColumn,
+                        StartIndex = left.StartIndex,
+                        EndIndex = right.EndIndex
+                    };
+                    break;
+                }
+                case SimpleExpressionToken.DIVIDE:
+                {
+                    result = new Result()
+                    {
+                        Value = left.Value / right.Value,
+                        StartColumn = left.StartColumn,
+                        EndColumn = right.EndColumn,
+                        StartIndex = left.StartIndex,
+                        EndIndex = right.EndIndex
+                    };
+                    break;
+                }
+            }
+
+            return result;
+        }
+
+
+        [Operation((int) SimpleExpressionToken.MINUS, Affix.PreFix, Associativity.Right, 100)]
+        public Result PreFixExpression(Token<SimpleExpressionToken> operation, Result value)
+        {
+            return new Result()
+            {
+                Value = -value.Value, 
+                StartColumn = operation.Position.Column,
+                EndColumn = value.EndColumn,
+                StartIndex = operation.Position.Index,
+                EndIndex = value.EndIndex
+            };
+        }
+
+
+
+        [Operand]
+        [Production("operand : primary_value")]
+        public Result OperandValue(Result value)
+        {
+            return value;
+        }
+
+
+        [Production("primary_value : DOUBLE")]
+        public Result OperandDouble(Token<SimpleExpressionToken> value)
+        {
+            return new Result()
+            {
+                Value = value.DoubleValue,
+                StartColumn = value.Position.Column,
+                EndColumn = value.Position.Column + value.Value.Length,
+                StartIndex = value.Position.Index,
+                EndIndex = value.Position.Index + value.Value.Length
+            };
+        }
+
+        [Production("primary_value : INT")]
+        public Result OperandInt(Token<SimpleExpressionToken> value)
+        {
+            return new Result()
+            {
+                Value = value.DoubleValue,
+                StartColumn = value.Position.Column,
+                EndColumn = value.Position.Column + value.Value.Length,
+                StartIndex = value.Position.Index,
+                EndIndex = value.Position.Index + value.Value.Length
+            };
+        }
+
+        [Production("primary_value : LPAREN Issue244Parser_expressions RPAREN")]
+        public Result OperandParens(Token<SimpleExpressionToken> lparen, Result value, Token<SimpleExpressionToken> rparen)
+        {
+            return new Result()
+            {
+                Value = value.Value,
+                StartColumn = lparen.Position.Column,
+                EndColumn = rparen.Position.Column + rparen.Value.Length,
+                StartIndex = rparen.Position.Index,
+                EndIndex = rparen.Position.Index + rparen.Value.Length,
+            };
+        }
+    }
+}
diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj
@@ -30,6 +30,7 @@
     <ProjectReference Include="..\indented\indented.csproj" />
     <ProjectReference Include="..\jsonparser\jsonparser.csproj" />
     <ProjectReference Include="..\..\sly\sly.csproj" />
+    <ProjectReference Include="..\postProcessedLexerParser\postProcessedLexerParser.csproj" />
     <ProjectReference Include="..\SimpleExpressionParser\SimpleExpressionParser.csproj" />
     <ProjectReference Include="..\while\while.csproj" />
   </ItemGroup>