Skip to content

Commit

Permalink
lexer post processing
Browse files Browse the repository at this point in the history
  • Loading branch information
@b3b00
b3b00 committed Aug 16, 2021
1 parent f929889 commit 6ea3af8
Show file tree
Hide file tree
Showing 13 changed files with 295 additions and 15 deletions.
6 changes: 3 additions & 3 deletions ParserTests/comments/CommentsErrorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ public void RedundantAttributes()
[Fact]
public void MixedErrors()
{
var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError10>>(),lang:"en");
var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError10>>(),lang: "en");
Assert.True(lexerRes10.IsError);
Assert.Equal(4, lexerRes10.Errors.Count);
var expectedErrors = new[]
Expand All @@ -199,7 +199,7 @@ public void MixedErrors()

expectedErrors = new[] {"too many multi-line comment lexem", "too many single-line comment lexem"};

var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError9>>(),lang:"en");
var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError9>>(),lang: "en");
Assert.True(lexerRes9.IsError);
Assert.Equal(2, lexerRes9.Errors.Count);
foreach (var expectedError in expectedErrors)
Expand All @@ -209,7 +209,7 @@ public void MixedErrors()

expectedErrors = new[] {"too many multi-line comment lexem","comment lexem can't be used together with single-line or multi-line comment lexems"};

var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError8>>(),lang:"en");
var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult<ILexer<CommentsTokenError8>>(),lang: "en");
Assert.True(lexerRes8.IsError);
Assert.Equal(2, lexerRes8.Errors.Count);
foreach (var expectedError in expectedErrors)
Expand Down
152 changes: 152 additions & 0 deletions samples/Issue244/Issue244Parser.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
using System;
using expressionparser;
using simpleExpressionParser;
using sly.lexer;
using sly.parser.generator;

namespace Issue244
{
public class Issue244Parser
{
[Operation((int) SimpleExpressionToken.PLUS, Affix.InFix, Associativity.Right, 10)]
[Operation("MINUS", Affix.InFix, Associativity.Left, 10)]
public Result BinaryTermExpression(Result left, Token<SimpleExpressionToken> operation, Result right)
{
// Get the length of the expression :
int length = right.EndIndex - left.StartIndex;


Result result = null;
switch (operation.TokenID)
{
case SimpleExpressionToken.PLUS:
{
result = new Result()
{
Value = left.Value + right.Value,
StartColumn = left.StartColumn,
EndColumn = right.EndColumn,
StartIndex = left.StartIndex,
EndIndex = right.EndIndex
}
;
break;
}
case SimpleExpressionToken.MINUS:
{
result = new Result()
{
Value = left.Value - right.Value,
StartColumn = left.StartColumn,
EndColumn = right.EndColumn,
StartIndex = left.StartIndex,
EndIndex = right.EndIndex
};
break;
}
}

return result;
}


[Operation((int) SimpleExpressionToken.TIMES, Affix.InFix, Associativity.Right, 50)]
[Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)]
public Result BinaryFactorExpression(Result left, Token<SimpleExpressionToken> operation, Result right)
{
Result result = null;
switch (operation.TokenID)
{
case SimpleExpressionToken.TIMES:
{
result = new Result()
{
Value = left.Value * right.Value,
StartColumn = left.StartColumn,
EndColumn = right.EndColumn,
StartIndex = left.StartIndex,
EndIndex = right.EndIndex
};
break;
}
case SimpleExpressionToken.DIVIDE:
{
result = new Result()
{
Value = left.Value / right.Value,
StartColumn = left.StartColumn,
EndColumn = right.EndColumn,
StartIndex = left.StartIndex,
EndIndex = right.EndIndex
};
break;
}
}

return result;
}


[Operation((int) SimpleExpressionToken.MINUS, Affix.PreFix, Associativity.Right, 100)]
public Result PreFixExpression(Token<SimpleExpressionToken> operation, Result value)
{
return new Result()
{
Value = -value.Value,
StartColumn = operation.Position.Column,
EndColumn = value.EndColumn,
StartIndex = operation.Position.Index,
EndIndex = value.EndIndex
};
}



[Operand]
[Production("operand : primary_value")]
public Result OperandValue(Result value)
{
return value;
}


[Production("primary_value : DOUBLE")]
public Result OperandDouble(Token<SimpleExpressionToken> value)
{
return new Result()
{
Value = value.DoubleValue,
StartColumn = value.Position.Column,
EndColumn = value.Position.Column + value.Value.Length,
StartIndex = value.Position.Index,
EndIndex = value.Position.Index + value.Value.Length
};
}

[Production("primary_value : INT")]
public Result OperandInt(Token<SimpleExpressionToken> value)
{
return new Result()
{
Value = value.DoubleValue,
StartColumn = value.Position.Column,
EndColumn = value.Position.Column + value.Value.Length,
StartIndex = value.Position.Index,
EndIndex = value.Position.Index + value.Value.Length
};
}

[Production("primary_value : LPAREN Issue244Parser_expressions RPAREN")]
public Result OperandParens(Token<SimpleExpressionToken> lparen, Result value, Token<SimpleExpressionToken> rparen)
{
return new Result()
{
Value = value.Value,
StartColumn = lparen.Position.Column,
EndColumn = rparen.Position.Column + rparen.Value.Length,
StartIndex = rparen.Position.Index,
EndIndex = rparen.Position.Index + rparen.Value.Length,
};
}
}
}
115 changes: 114 additions & 1 deletion samples/ParserExample/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using csly.whileLang.model;
using csly.whileLang.parser;
using expressionparser;
using expressionparser.model;
using GenericLexerWithCallbacks;
using indented;
using jsonparser;
Expand Down Expand Up @@ -928,7 +929,8 @@ private static void Main(string[] args)
// TestI18N();
// Console.ReadLine();
// TestShortGeneric();
TestIssue239();
//TestIssue239();
TestLexerPostProcess();
}


Expand Down Expand Up @@ -972,6 +974,117 @@ private static void TestIssue239()
Issue239Tests.TestOk();
;
}


private static List<Token<ExpressionToken>> postProcess(List<Token<ExpressionToken>> tokens)
{
var mayLeft = new List<ExpressionToken>()
{
ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.IDENTIFIER
};

var mayRight = new List<ExpressionToken>()
{
ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.LPAREN, ExpressionToken.IDENTIFIER
};

Func<ExpressionToken,bool> mayOmmitLeft = (ExpressionToken tokenid) => mayLeft.Contains(tokenid);

Func<ExpressionToken,bool> mayOmmitRight = (ExpressionToken tokenid) => mayRight.Contains(tokenid);


List<Token<ExpressionToken>> newTokens = new List<Token<ExpressionToken>>();
for (int i = 0; i < tokens.Count; i++)
{
if ( i >= 1 &&
mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID))
{
newTokens.Add(new Token<ExpressionToken>()
{
TokenID = ExpressionToken.TIMES
});
}
newTokens.Add(tokens[i]);
}

return newTokens;
}

private static void TestLexerPostProcess()
{
var parserInstance = new VariableExpressionParser();
var builder = new ParserBuilder<ExpressionToken, expressionparser.model.Expression>();
var build = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression",
lexerPostProcess: postProcess);
if (build.IsError)
{
foreach (var error in build.Errors)
{
Console.WriteLine(error.Message);
}

return;
}

var Parser = build.Result;
var r = Parser.Parse("2 * x");
if (r.IsError)
{
foreach (var error in r.Errors)
{
Console.WriteLine(error.ErrorMessage);
}

return;
}
var res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, int>()
{ { "x", 2 } }));
Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?"));


r = Parser.Parse("2 x");
if (r.IsError)
{
foreach (var error in r.Errors)
{
Console.WriteLine(error.ErrorMessage);
}

return;
}
res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, int>()
{ { "x", 2 } }));
Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?"));

r = Parser.Parse("2 ( x ) ");
if (r.IsError)
{
foreach (var error in r.Errors)
{
Console.WriteLine(error.ErrorMessage);
}

return;
}
res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, int>()
{ { "x", 2 } }));
Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?"));

r = Parser.Parse("x x ");
if (r.IsError)
{
foreach (var error in r.Errors)
{
Console.WriteLine(error.ErrorMessage);
}

return;
}
res = r.Result.Evaluate(new ExpressionContext(new Dictionary<string, int>()
{ { "x", 2 } }));
Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?"));

}
}

public enum TestGrammarToken
Expand Down
3 changes: 2 additions & 1 deletion samples/jsonparser/JSONLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ namespace jsonparser
public class JSONLexer : ILexer<JsonToken>
{
public string I18n { get; set; }

public LexerPostProcess<JsonToken> LexerPostProcess { get; set; }

public void AddDefinition(TokenDefinition<JsonToken> tokenDefinition)
{
}
Expand Down
2 changes: 2 additions & 0 deletions sly/lexer/GenericLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ public Config()
public IEqualityComparer<string> KeyWordComparer => KeyWordIgnoreCase ? StringComparer.OrdinalIgnoreCase : null;
}

public LexerPostProcess<IN> LexerPostProcess { get; set; }

public string I18n { get; set; }

public const string in_string = "in_string";
Expand Down
3 changes: 3 additions & 0 deletions sly/lexer/ILexer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

using System;
using sly.lexer.fsm;

namespace sly.lexer
{
Expand All @@ -11,5 +12,7 @@ public interface ILexer<T> where T : struct
LexerResult<T> Tokenize(ReadOnlyMemory<char> source);

string I18n { get; set; }

LexerPostProcess<T> LexerPostProcess { get; set; }
}
}
4 changes: 3 additions & 1 deletion sly/lexer/Lexer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using sly.lexer.fsm;

namespace sly.lexer
{
Expand All @@ -10,7 +11,8 @@ namespace sly.lexer
public class Lexer<T> : ILexer<T> where T : struct
{
public string I18n { get; set; }

public LexerPostProcess<T> LexerPostProcess { get; set; }

private readonly IList<TokenDefinition<T>> tokenDefinitions = new List<TokenDefinition<T>>();

public void AddDefinition(TokenDefinition<T> tokenDefinition)
Expand Down
4 changes: 2 additions & 2 deletions sly/lexer/LexerBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,14 @@ public static BuildResult<ILexer<IN>> BuildLexer<IN>(BuildExtension<IN> extensio

public static BuildResult<ILexer<IN>> BuildLexer<IN>(BuildResult<ILexer<IN>> result,
BuildExtension<IN> extensionBuilder = null,
string lang = null) where IN : struct
string lang = null, LexerPostProcess<IN> lexerPostProcess = null) where IN : struct
{
var attributes = GetLexemes(result,lang);
if (!result.IsError)
{
result = Build(attributes, result, extensionBuilder,lang);
}

result.Result.LexerPostProcess = lexerPostProcess;
return result;
}

Expand Down
2 changes: 2 additions & 0 deletions sly/lexer/fsm/FSMLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ public static T At<T>(this ReadOnlyMemory<T> memory, LexerPosition position)
}

public delegate void BuildExtension<IN>(IN token, LexemeAttribute lexem, GenericLexer<IN> lexer) where IN : struct;

public delegate List<Token<IN>> LexerPostProcess<IN>(List<Token<IN>> tokens) where IN : struct;

public class FSMLexer<N>
{
Expand Down
Loading

0 comments on commit 6ea3af8

Please sign in to comment.