Skip to content

Commit

Permalink
Merge branch 'dev' of github.com:b3b00/csly into dev
Browse files Browse the repository at this point in the history 
* 'dev' of github.com:b3b00/csly:
  Update README.md
  Update README.md
  #165 fix issues after @CP3088 pull-request
  Optimized Expression Rules for #165
  looking at #165
  • Loading branch information
@b3b00
b3b00 committed Apr 15, 2020
2 parents 08dcc50 + c8f8843 commit 46d803e
Show file tree
Hide file tree
Showing 8 changed files with 216 additions and 62 deletions.
2 changes: 1 addition & 1 deletion ParserTests/EBNFTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,7 @@ public void TestZeroOrMoreWithOne()

#region CONTEXTS

private BuildResult<Parser<ExpressionToken, int>> buildSimpleExpressionParserWithContext(ParserType parserType = ParserType.LL_RECURSIVE_DESCENT)
private BuildResult<Parser<ExpressionToken, int>> buildSimpleExpressionParserWithContext(ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT)
{
var startingRule = $"{typeof(SimpleExpressionParserWithContext).Name}_expressions";
var parserInstance = new SimpleExpressionParserWithContext();
Expand Down
6 changes: 3 additions & 3 deletions ParserTests/ExpressionGeneratorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ private void BuildParser()
StartingRule = $"{typeof(SimpleExpressionParser).Name}_expressions";
var parserInstance = new SimpleExpressionParser();
var builder = new ParserBuilder<ExpressionToken, double>();
Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, StartingRule);
Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule);
}

[Fact]
Expand Down Expand Up @@ -76,12 +76,12 @@ public void TestBuild()
Assert.Equal(3, nt.Rules.Count);
Assert.Contains("primary_value", nt.Name);
nt = nonterminals[2];
Assert.Equal(3, nt.Rules.Count);
Assert.Equal(2, nt.Rules.Count);
Assert.Contains("10", nt.Name);
Assert.Contains("PLUS", nt.Name);
Assert.Contains("MINUS", nt.Name);
nt = nonterminals[3];
Assert.Equal(3, nt.Rules.Count);
Assert.Equal(2, nt.Rules.Count);
Assert.Contains("50", nt.Name);
Assert.Contains("TIMES", nt.Name);
Assert.Contains("DIVIDE", nt.Name);
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# C# Lex Yacc #

[![Build status](https://ci.appveyor.com/api/projects/status/n9uffgkqn2qet7k9?svg=true)](https://ci.appveyor.com/project/OlivierDuhart/sly)
![AppVeyor tests (compact)](https://img.shields.io/appveyor/tests/OlivierDuhart/sly.svg?compact_message)
![Test status](http://teststatusbadge.azurewebsites.net/api/status/mmaitre314/securestringcodegen)
[![coveralls](https://coveralls.io/repos/github/b3b00/csly/badge.svg?branch=dev)](https://coveralls.io/github/b3b00/csly?branch=dev)
![.NET Core](https://github.com/b3b00/csly/workflows/.NET%20Core/badge.svg)
[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fb3b00%2Fcsly.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fb3b00%2Fcsly?ref=badge_shield)
Expand Down
50 changes: 50 additions & 0 deletions samples/ParserExample/Issue165Lexer.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
using sly.lexer;

namespace ParserExample
{
public enum Issue165Lexer
{
[Lexeme(GenericToken.SugarToken,"?")]
QUESTIONMARK = 1,

[Lexeme(GenericToken.SugarToken,"+")]
PLUS = 2,

[Lexeme(GenericToken.SugarToken,"-")]
MINUS = 3,

[Lexeme(GenericToken.SugarToken,"*")]
TIMES = 4,

[Lexeme(GenericToken.SugarToken,"/")]
DIVIDE = 5,

[Lexeme(GenericToken.SugarToken,"(")]
LPAR = 6,

[Lexeme(GenericToken.SugarToken,")")]
RPAR = 6,

[Lexeme(GenericToken.SugarToken,"[")]
LBR = 7,

[Lexeme(GenericToken.SugarToken,"]")]
RBR = 8,

[Lexeme(GenericToken.SugarToken,":")]
COLON = 7,

[Lexeme(GenericToken.SugarToken,"=")]
EQ = 8,


[Lexeme(GenericToken.Identifier)] ID = 20,

[Lexeme(GenericToken.String)] STRING = 21,

[Lexeme(GenericToken.Int)] INT = 22,



}
}
88 changes: 88 additions & 0 deletions samples/ParserExample/Issue165Parser.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
using expressionparser;
using sly.lexer;
using sly.parser.generator;

namespace ParserExample
{
public class Issue165Parser
{

[Production("test_stmt : ID EQ [d] arith_expr ")]
public object assign(Token<Issue165Lexer> id, object expr)
{
return null;
}



[Operation((int) Issue165Lexer.PLUS, Affix.InFix, Associativity.Right, 10)]
[Operation("MINUS", Affix.InFix, Associativity.Left, 10)]
public object BinaryTermExpression(object left, Token<Issue165Lexer> operation, object right)
{
return null;
}


[Operation((int) Issue165Lexer.TIMES, Affix.InFix, Associativity.Right, 50)]
[Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)]
public object BinaryFactorExpression(double left, Token<Issue165Lexer> operation, object right)
{

return null;
}



[Production("arith_expr : Issue165Parser_expressions")]
[Production("arith_expr : ternary_expr")]
public object arith(object h)
{
return h;
}

[Production(
"ternary_expr : Issue165Parser_expressions QUESTIONMARK [d] Issue165Parser_expressions COLON [d] Issue165Parser_expressions")]
public object ternary(object j, object k, object l)
{
return null;
}

[Production("sqbr_expr : ID LBR [d] Issue165Parser_expressions RBR [d]")]
public object sqrbrack(Token<Issue165Lexer> token, object expr)
{
return null;
}

[Production("group_expr : LPAR [d] Issue165Parser_expressions RPAR [d]")]
public object group(object subexpr)
{
return subexpr;
}

[Operand]
[Production("atom : ID")]
public object id(Token<Issue165Lexer> i)
{
return null;
}
[Operand]
[Production("atom : INT")]
public object integer(Token<Issue165Lexer> i)
{
return i;
}
[Operand]
[Production("atom : sqbr_expr")]
public object sq(object i)
{
return i;
}
[Operand]
[Production("atom : group_expr")]
public object id(object g)
{
return g;
}

}
}
113 changes: 64 additions & 49 deletions sly/parser/generator/ExpressionRulesGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,16 @@ public static BuildResult<ParserConfiguration<IN, OUT>> BuildExpressionRules<IN,
return attr != null;
}).ToList();


var operationsByPrecedence = new Dictionary<int, List<OperationMetaData<IN>>>();


methods.ForEach(m =>
{
var attributes =
(OperationAttribute[]) m.GetCustomAttributes(typeof(OperationAttribute), true);
(OperationAttribute[])m.GetCustomAttributes(typeof(OperationAttribute), true);

foreach (var attr in attributes)
{
IN oper = default(IN);
IN oper = default;
if (attr.IsIntToken)
{
oper = EnumConverter.ConvertIntToEnum<IN>(attr.IntToken);
Expand All @@ -74,7 +72,7 @@ public static BuildResult<ParserConfiguration<IN, OUT>> BuildExpressionRules<IN,
{
oper = EnumConverter.ConvertStringToEnum<IN>(attr.StringToken);
}
var operation = new OperationMetaData<IN>(attr.Precedence, attr.Assoc, m, attr.Affix,oper);
var operation = new OperationMetaData<IN>(attr.Precedence, attr.Assoc, m, attr.Affix, oper);
var operations = new List<OperationMetaData<IN>>();
if (operationsByPrecedence.ContainsKey(operation.Precedence))
operations = operationsByPrecedence[operation.Precedence];
Expand Down Expand Up @@ -127,19 +125,16 @@ private static void GenerateExpressionParser<IN, OUT>(ParserConfiguration<IN, OU
{
var precedences = operationsByPrecedence.Keys.ToList();
precedences.Sort();
var max = precedences.Max();

for (var i = 0; i < precedences.Count; i++)
{
var precedence = precedences[i];
var nextPrecedence = i < precedences.Count - 1 ? precedences[i + 1] : -1;
var operations = operationsByPrecedence[precedence];
var name = GetNonTerminalNameForPrecedence(precedence, operationsByPrecedence, operandNonTerminal);
var nextName =
GetNonTerminalNameForPrecedence(nextPrecedence, operationsByPrecedence, operandNonTerminal);
var nextName = GetNonTerminalNameForPrecedence(nextPrecedence, operationsByPrecedence, operandNonTerminal);

var nonTerminal = BuildNonTerminal(i == precedences.Count - 1, name, nextName, operations,
operationsByPrecedence);
var nonTerminal = BuildNonTerminal(name, nextName, operations);

configuration.NonTerminals[nonTerminal.Name] = nonTerminal;
}
Expand All @@ -157,44 +152,69 @@ private static void GenerateExpressionParser<IN, OUT>(ParserConfiguration<IN, OU
entrypoint.Rules.Add(rule);
}


private static NonTerminal<IN> BuildNonTerminal<IN>(bool last, string name, string nextName,
List<OperationMetaData<IN>> operations, Dictionary<int, List<OperationMetaData<IN>>> operationsByPrecedence)
where IN : struct
private static NonTerminal<IN> BuildNonTerminal<IN>(string name, string nextName, List<OperationMetaData<IN>> operations) where IN : struct
{
var nonTerminal = new NonTerminal<IN>(name, new List<Rule<IN>>());
foreach (var operation in operations)
if (operation.Affix == Affix.InFix)

var InFixOps = operations.Where(x => x.Affix == Affix.InFix).ToList();
if (InFixOps.Count > 0)
{
var InFixClauses = InFixOps.Select(x => new TerminalClause<IN>(x.OperatorToken)).ToList<IClause<IN>>();

var rule = new Rule<IN>()
{
var rule = new Rule<IN>();
rule.Clauses.Add(new NonTerminalClause<IN>(nextName));
rule.Clauses.Add(new TerminalClause<IN>(operation.OperatorToken));
rule.Clauses.Add(new NonTerminalClause<IN>(name));
rule.IsExpressionRule = true;
rule.ExpressionAffix = operation.Affix;
rule.SetVisitor(operation);
nonTerminal.Rules.Add(rule);
}
else if (operation.Affix == Affix.PreFix)
ExpressionAffix = Affix.InFix,
IsExpressionRule = true
};

rule.Clauses.Add(new NonTerminalClause<IN>(nextName));
rule.Clauses.Add(InFixClauses.Count == 1 ? InFixClauses[0] : new ChoiceClause<IN>(InFixClauses));
rule.Clauses.Add(new NonTerminalClause<IN>(name));

InFixOps.ForEach(x =>
{
var rule = new Rule<IN>();
rule.Clauses.Add(new TerminalClause<IN>(operation.OperatorToken));
rule.Clauses.Add(new NonTerminalClause<IN>(nextName));
rule.SetVisitor(x);
rule.IsExpressionRule = true;
rule.ExpressionAffix = operation.Affix;
rule.SetVisitor(operation);
nonTerminal.Rules.Add(rule);
}
else if (operation.Affix == Affix.PostFix)
});
nonTerminal.Rules.Add(rule);
}


var PreFixOps = operations.Where(x => x.Affix == Affix.PreFix).ToList();
if (PreFixOps.Count > 0)
{
var PreFixClauses = PreFixOps.Select(x => new TerminalClause<IN>(x.OperatorToken)).ToList<IClause<IN>>();

var rule = new Rule<IN>()
{
var rule = new Rule<IN>();
rule.Clauses.Add(new NonTerminalClause<IN>(nextName));
rule.Clauses.Add(new TerminalClause<IN>(operation.OperatorToken));
rule.IsExpressionRule = true;
rule.ExpressionAffix = operation.Affix;
rule.SetVisitor(operation);
nonTerminal.Rules.Add(rule);
}
ExpressionAffix = Affix.PreFix,
IsExpressionRule = true
};

rule.Clauses.Add(PreFixClauses.Count == 1 ? PreFixClauses[0] : new ChoiceClause<IN>(PreFixClauses));
rule.Clauses.Add(new NonTerminalClause<IN>(nextName));

PreFixOps.ForEach(x => rule.SetVisitor(x));
nonTerminal.Rules.Add(rule);
}

var PostFixOps = operations.Where(x => x.Affix == Affix.PostFix).ToList();
if (PostFixOps.Count > 0)
{
var PostFixClauses = PostFixOps.Select(x => new TerminalClause<IN>(x.OperatorToken)).ToList<IClause<IN>>();

var rule = new Rule<IN>()
{
ExpressionAffix = Affix.PostFix,
IsExpressionRule = true
};

rule.Clauses.Add(new NonTerminalClause<IN>(nextName));
rule.Clauses.Add(PostFixClauses.Count == 1 ? PostFixClauses[0] : new ChoiceClause<IN>(PostFixClauses));

PostFixOps.ForEach(x => rule.SetVisitor(x));
nonTerminal.Rules.Add(rule);
}

var rule0 = new Rule<IN>();
rule0.Clauses.Add(new NonTerminalClause<IN>(nextName));
Expand Down Expand Up @@ -224,13 +244,8 @@ private static string GetNonTerminalNameForPrecedence<IN>(int precedence, List<I
.Select(oper => oper.ToString())
.ToList()
.Aggregate((s1, s2) => $"{s1}_{s2}");
var name = $"expr_{precedence}_{operatorsPart}";


return name;

return $"expr_{precedence}_{operatorsPart}";
}



}
}
11 changes: 6 additions & 5 deletions sly/parser/syntax/grammar/Rule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@ public string Key
{
get
{
var k = Clauses
.Select(c => c.ToString())
.Aggregate((c1, c2) => c1.ToString() + "_" + c2.ToString());
if (Clauses.Count == 1) k += "_";
return k;
var key = string.Join("_", Clauses.Select(c => c.ToString()));

if (Clauses.Count == 1)
key += "_";

return IsExpressionRule ? key.Replace(" | ", "_") : key;
}
}

Expand Down
6 changes: 3 additions & 3 deletions sly/parser/syntax/tree/SyntaxNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public class SyntaxNode<IN> : ISyntaxNode<IN> where IN : struct
public SyntaxNode(string name, List<ISyntaxNode<IN>> children = null, MethodInfo visitor = null)
{
Name = name;
Children = children == null ? new List<ISyntaxNode<IN>>() : children;
Children = children ?? new List<ISyntaxNode<IN>>();
Visitor = visitor;
}

Expand All @@ -34,8 +34,8 @@ public SyntaxNode(string name, List<ISyntaxNode<IN>> children = null, MethodInfo

public bool IsExpressionNode => Operation != null;

public bool IsBinaryOperationNode => IsExpressionNode ? Operation.Affix == Affix.InFix : false;
public bool IsUnaryOperationNode => IsExpressionNode ? Operation.Affix != Affix.InFix : false;
public bool IsBinaryOperationNode => IsExpressionNode && Operation.Affix == Affix.InFix;
public bool IsUnaryOperationNode => IsExpressionNode && Operation.Affix != Affix.InFix;
public int Precedence => IsExpressionNode ? Operation.Precedence : -1;

public Associativity Associativity =>
Expand Down

0 comments on commit 46d803e

Please sign in to comment.