Merge a992f33 into 1a66abe

ParserTests/Issue259/Issue259ExpressionToken.cs

@@ -0,0 +1,68 @@
+using sly.lexer;
+
+namespace CslyNullIssue
+{
+    public enum Issue259ExpressionToken
+    {
+        [Lexeme("L:[A-Za-z0-9_]+")]
+        LVAR = 50,
+
+        [Lexeme("A:[:A-Za-z0-9 ]+,\\s*([A-Za-z0-9 ]+)")]
+        SIMVAR = 52,
+
+        // [Lexeme("\\[[A-Z_0-9 ]+:[A-Z_0-9 ]+\\]")]
+        [Lexeme("\\[[A-Za-z_0-9 ]+:[A-Za-z_0-9 ]+\\]")]
+        DCS_VAR = 53,
+
+        [Lexeme("OFF")]
+        OFF = 0,
+
+        [Lexeme("ON")]
+        ON = 1,
+
+        // Hex/decimal prefix?
+        [Lexeme("0[xX][0-9a-fA-F]+")]
+        HEX_NUMBER = 2,
+
+        [Lexeme("[0-9]+(\\.[0-9]+)?")]
+        DECIMAL_NUMBER = 3,
+
+        [Lexeme("\\+")]
+        PLUS = 4,
+
+        [Lexeme("-")]
+        MINUS = 5,
+
+        [Lexeme("\\*")]
+        TIMES = 6,
+
+        [Lexeme("/")]
+        DIVIDE = 7,
+
+        [Lexeme("\\|")]
+        BITWISE_OR = 8,
+
+        [Lexeme("&")]
+        BITWISE_AND = 9,
+
+        [Lexeme("[ \\t]+", isSkippable: true)]
+        WHITESPACE = 20,
+
+        [Lexeme("OR")]
+        LOGICAL_OR = 10,
+        [Lexeme("AND")]
+        LOGICAL_AND = 11,
+
+        [Lexeme("NOT")]
+        NOT = 12,
+
+        [Lexeme("(<=?)|(==)|(!=)|(>=?)")]
+        COMPARISON = 13,
+
+        [Lexeme("\\(")]
+        LPAREN = 30,
+
+        [Lexeme("\\)")]
+        RPAREN = 31,
+    }
+}

ParserTests/Issue259/Issue259Parser.cs

@@ -0,0 +1,97 @@
+using System.Globalization;
+using sly.lexer;
+using sly.parser.generator;
+
+namespace CslyNullIssue
+{
+    public class Issue259Parser 
+    {
+        [Operand]
+        [Production("logical_literal: OFF")]
+        [Production("logical_literal: ON")]
+        public string LiteralBool(Token<Issue259ExpressionToken> token)
+        {
+            return token.Value;
+        }
+
+        [Operand]
+        [Production("primary: HEX_NUMBER")]
+        public string NumericExpressionFromLiteralNumber(Token<Issue259ExpressionToken> offsetToken)
+        {
+            return offsetToken.Value;
+        }
+
+        [Operand]
+        [Production("primary: DECIMAL_NUMBER")]
+        public string NumericExpressionFromDecimalNumber(Token<Issue259ExpressionToken> offsetToken)
+        {
+            var text = offsetToken.Value;
+            var value = double.Parse(text, CultureInfo.InvariantCulture);
+            return value.ToString(CultureInfo.InvariantCulture);
+        }
+
+        [Infix((int)Issue259ExpressionToken.PLUS, Associativity.Left, 14)]
+        [Infix((int)Issue259ExpressionToken.MINUS, Associativity.Left, 14)]
+        [Infix((int)Issue259ExpressionToken.TIMES, Associativity.Left, 15)]
+        [Infix((int)Issue259ExpressionToken.DIVIDE, Associativity.Left, 15)]
+        [Infix((int)Issue259ExpressionToken.BITWISE_AND, Associativity.Left, 10)]
+        [Infix((int)Issue259ExpressionToken.BITWISE_OR, Associativity.Left, 8)]
+        public string NumberExpression(string lhs, Token<Issue259ExpressionToken> token, string rhs)
+        {
+            return $"({lhs} {token.Value} {rhs})";
+        }
+
+        [Infix((int)Issue259ExpressionToken.LOGICAL_AND, Associativity.Left, 7)]
+        [Infix((int)Issue259ExpressionToken.LOGICAL_OR, Associativity.Left, 6)]
+        public string LogicalExpression(string lhs, Token<Issue259ExpressionToken> token, string rhs)
+        {
+            return $"({lhs} {token.Value} {rhs})";
+        }
+
+        [Prefix((int)Issue259ExpressionToken.MINUS, Associativity.Right, 17)]
+        public string NumericExpression(Token<Issue259ExpressionToken> _, string child)
+        {
+            return $"-{child}";
+        }
+
+        // We want NOT to to bind tighter than AND/OR but looser than numeric comparison operations
+        [Prefix((int)Issue259ExpressionToken.NOT, Associativity.Right, 11)]
+        public string LogicalExpression(Token<Issue259ExpressionToken> _, string child)
+        {
+            return $"(NOT {child})";
+        }
+
+        [Infix((int)Issue259ExpressionToken.COMPARISON, Associativity.Left, 12)]
+        public string Comparison(string lhs, Token<Issue259ExpressionToken> token, string rhs)
+        {
+            return $"({lhs} {token.Value} {rhs})";
+        }
+
+
+        [Operand]
+        [Production("numeric_literal: LVAR")]
+        public string Lvar(Token<Issue259ExpressionToken> token)
+        {
+            return token.Value;
+        }
+
+        [Operand]
+        [Production("numeric_literal: SIMVAR")]
+        public string SimVarExpression(Token<Issue259ExpressionToken> simvarToken)
+        {
+            var text = simvarToken.Value[2..];
+            var bits = text.Split(",");
+            var varName = bits[0];
+            var type = bits[1].Trim();
+            return $"A:{varName}, {type}";
+        }
+
+        [Operand]
+        [Production("group : LPAREN Issue259Parser_expressions RPAREN")]
+        public string Group(Token<Issue259ExpressionToken> _1, string child, Token<Issue259ExpressionToken> _2)
+        {
+            return child;
+        }
+
+    }
+}

ParserTests/Issue259/Issue259Tests.cs

@@ -0,0 +1,47 @@
+using System;
+using sly.parser;
+using sly.parser.generator;
+using Xunit;
+
+namespace CslyNullIssue
+{
+    public class Issue259Tests
+    {
+        [Fact]
+        public static void Issue259Test()
+        {
+            var expression = "1 < 2 AND 3 <= 4 AND 5 == 6 AND 7 >= 8 AND 9 > 10 AND 11 != 12 AND 13 <> 14";
+
+
+            var startingRule = $"{typeof(Issue259Parser).Name}_expressions";
+            var parserInstance = new Issue259Parser();
+            var builder = new ParserBuilder<Issue259ExpressionToken, string>("en");
+            var parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, startingRule);
+            Assert.True(parser.IsOk);
+            Assert.NotNull(parser.Result);
+
+            Console.WriteLine(parser.Result.Configuration.Dump());
+
+            var parseResult = parser.Result.Parse(expression);
+
+            Assert.True(parseResult.IsError);
+            Assert.Single(parseResult.Errors);
+
+            var error = parseResult.Errors[0];
+
+            var unexpectedTokenError = error as UnexpectedTokenSyntaxError<Issue259ExpressionToken>;
+            Assert.NotNull(unexpectedTokenError);
+            Assert.Equal(Issue259ExpressionToken.COMPARISON, unexpectedTokenError.UnexpectedToken.TokenID);
+            Assert.Equal(">",unexpectedTokenError.UnexpectedToken.Value);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.ON);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.OFF);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.MINUS);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.HEX_NUMBER);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.DECIMAL_NUMBER);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.LVAR);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.SIMVAR);
+            Assert.Contains(unexpectedTokenError.ExpectedTokens, x => x == Issue259ExpressionToken.LPAREN);
+
+        }
+    }
+}

sly/parser/generator/ExpressionRulesGenerator.cs

@@ -223,7 +223,8 @@ private NonTerminal<IN> BuildPrecedenceNonTerminal(string name, string nextName,
                 var rule = new Rule<IN>()
                 {
                     ExpressionAffix = Affix.InFix,
-                    IsExpressionRule = true
+                    IsExpressionRule = true,
+                    NonTerminalName = name
                 };
 
                 rule.Clauses.Add(new NonTerminalClause<IN>(nextName));

sly/parser/generator/NonTerminal.cs

@@ -33,7 +33,8 @@ public string Dump()
 
             foreach (var rule in Rules)
             {
-                dump.Append(Name).Append(" : ");
+
+                dump.Append(Name).Append(rule.IsInfixExpressionRule ? " (*) ":"").Append(" : ");
                 foreach (IClause<IN> clause in rule.Clauses)
                 {
                     dump.Append(clause.Dump()).Append(" ");

sly/parser/parser/Parser.cs

@@ -119,9 +119,14 @@ public ParseResult<IN, OUT> ParseWithContext(IList<Token<IN>> tokens, object par
                 var errors = new List<ParseError>();  
                 foreach (var expecting in byEnding)
                 {
-                    var expectingTokens = expecting.SelectMany(x => x.ExpectedTokens).Distinct(); 
-                    var expected = new UnexpectedTokenSyntaxError<IN>(expecting.First().UnexpectedToken, I18n,expectingTokens.ToArray());
-                    errors.Add(expected);
+                    var expectingTokens = expecting.SelectMany(x => (x.ExpectedTokens ?? new List<IN>())).Distinct();
+                    var expectedTokens =  (expectingTokens != null && expectingTokens.Any()) ? expectingTokens?.ToArray() : null;
+                    if (expectedTokens != null)
+                    {
+                        var expected = new UnexpectedTokenSyntaxError<IN>(expecting.First().UnexpectedToken, I18n,
+                            expectedTokens);
+                        errors.Add(expected);
+                    }
                 }
 
                 result.Errors.AddRange(errors);

sly/parser/parser/SyntaxParseResult.cs

@@ -30,6 +30,10 @@ public void AddExpecting(IN expected)
 
         public void AddExpectings(IEnumerable<IN> expected)
         {
+            if (expected == null)
+            {
+                return;
+            }
             if (Expecting == null)
             {
                 Expecting = new List<IN>();

sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs

@@ -184,12 +184,6 @@ public virtual SyntaxParseResult<IN> ParseInfixExpressionRule(IList<Token<IN>> t
                             {
                                 if (firstResult.Root is SyntaxNode<IN> node)
                                 {
-                                    if (node.Name == "expr_17_MINUS")
-                                    {
-                                        ;
-                                    }
-                                    // node.IsByPassNode = true;
-                                    // node.HasByPassNodes = true;
                                     firstResult.Errors.AddRange(secondResult.Errors);
                                     firstResult.AddExpectings(secondResult.Expecting);
                                     return firstResult;
@@ -225,14 +219,9 @@ public virtual SyntaxParseResult<IN> ParseInfixExpressionRule(IList<Token<IN>> t
                         if (third is NonTerminalClause<IN> thirdNonTerminal)
                         {
                             thirdResult = ParseNonTerminal(tokens, thirdNonTerminal, currentPosition);
-
                             if (thirdResult.IsError)
                             {
-                                if (firstResult.Root is SyntaxNode<IN> node)
-                                {
-                                    firstResult.AddExpectings(thirdResult.Expecting);
-                                    return firstResult;
-                                }
+                                return thirdResult;
                             }
                             else
                             {

sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs

@@ -239,6 +239,7 @@ public SyntaxParseResult<IN> ParseTerminal(IList<Token<IN>> tokens, TerminalClau
             token.Discarded = terminal.Discarded;
             result.Root = new SyntaxLeaf<IN>(token, terminal.Discarded);
             result.HasByPassNodes = false;
+            result.Errors.Add(new UnexpectedTokenSyntaxError<IN>(token,I18n,terminal.ExpectedToken));
             result.AddExpecting(terminal.ExpectedToken);
             return result;
         }
@@ -396,6 +397,7 @@ private SyntaxParseResult<IN> NoMatchingRuleError(IList<Token<IN>> tokens, int c
             error.IsEnded = false;
             error.Errors = noRuleErrors;
             error.EndingPosition = currentPosition;
+            error.Expecting = allAcceptableTokens;
 
             return error;
         }

sly/sly.csproj

@@ -6,11 +6,11 @@
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
     <Description>#LY is a parser generator halfway between parser combinators and parser generator like ANTLR</Description>
     <Authors>b3b00</Authors>
-    <version>2.8.0</version>
+    <version>2.8.0.1</version>
     <PackageProjectUrl>https://github.com/b3b00/sly</PackageProjectUrl>
     <RepositoryUrl>https://github.com/b3b00/sly</RepositoryUrl>
     <PackageLicenseUrl>https://github.com/b3b00/sly/blob/master/LICENSE</PackageLicenseUrl>
-    <PackageVersion>2.8.0</PackageVersion>
+    <PackageVersion>2.8.0.1</PackageVersion>
     <ApplicationIcon />
     <OutputType>Library</OutputType>
     <StartupObject />