diff --git a/ParserTests/EBNFTests.cs b/ParserTests/EBNFTests.cs index f48d7298..35885c6b 100644 --- a/ParserTests/EBNFTests.cs +++ b/ParserTests/EBNFTests.cs @@ -1,5 +1,6 @@ using System.Linq; using Xunit; +using System.Text; using sly.parser; using sly.lexer; using sly.parser.generator; @@ -45,11 +46,32 @@ public string root(Token a, string b, Token c) else { r = $"{r},{c.Value})"; } - return r; } - [Production("B : b ")] + [Production("root : a b? c ")] + public string root2(Token a, Token b, Token c) + { + StringBuilder result = new StringBuilder(); + result.Append("R("); + result.Append(a.StringWithoutQuotes); + result.Append(","); + if (b.IsEmpty) + { + result.Append(""); + } + else + { + result.Append(b.StringWithoutQuotes); + } + result.Append(","); + result.Append(c.StringWithoutQuotes); + result.Append(")"); + + return result.ToString(); + } + + [Production("B : b ")] public string bee(Token b) { return $"B({b.Value})"; } @@ -321,6 +343,17 @@ public void TestEmptyOption() { Assert.Equal("R(a,B(b),)", result.Result); } + [Fact] + public void TestEmptyOptionInMiddle() + { + var buildResult = BuildOptionParser(); + Assert.False(buildResult.IsError); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a c"); + Assert.Equal("R(a,,c)", result.Result); + } + private void AssertString(JObject obj, string key, string value) { Assert.True(obj.ContainsKey(key)); diff --git a/ParserTests/ErrorTests.cs b/ParserTests/ErrorTests.cs index 6655e8f0..62bcc723 100644 --- a/ParserTests/ErrorTests.cs +++ b/ParserTests/ErrorTests.cs @@ -21,9 +21,9 @@ public void TestJsonSyntaxError() string source = @"{ - 'one': 1, - 'bug':{,} - }".Replace("'", "\""); + 'one': 1, + 'bug':{,} +}".Replace("'", "\""); ParseResult r = parser.Parse(source); Assert.True(r.IsError); Assert.Null(r.Result); @@ -33,8 +33,8 @@ public void TestJsonSyntaxError() UnexpectedTokenSyntaxError error = r.Errors[0] as UnexpectedTokenSyntaxError; Assert.Equal(JsonToken.COMMA, error?.UnexpectedToken.TokenID); - Assert.Equal(3, error?.Line); - Assert.Equal(24, error?.Column); + Assert.Equal(2, error?.Line); + Assert.Equal(13, error?.Column); } @@ -49,13 +49,13 @@ public void TestExpressionSyntaxError() Assert.True(r.IsError); Assert.NotNull(r.Errors); Assert.True(r.Errors.Count > 0); - Assert.IsAssignableFrom(typeof(UnexpectedTokenSyntaxError), r.Errors[0]); + Assert.IsAssignableFrom < UnexpectedTokenSyntaxError>(r.Errors[0]); UnexpectedTokenSyntaxError error = r.Errors[0] as UnexpectedTokenSyntaxError; Assert.Equal(ExpressionToken.PLUS, error.UnexpectedToken.TokenID); Assert.Equal(1, error.Line); - Assert.Equal(10, error.Column); + Assert.Equal(8, error.Column); } [Fact] diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index fb2f3b96..c456038c 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -331,37 +331,35 @@ static void testJSONLexer() ; } - static void Main(string[] args) + + static void testErrors() { + JSONParser jsonParser = new JSONParser(); + ParserBuilder builder = new ParserBuilder(); + Parser parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root").Result; + + + string source = @"{ + 'one': 1, + 'bug':{,} +}".Replace("'", "\""); + ParseResult r = parser.Parse(source); + + bool isError = r.IsError; // true + var root = r.Result; // null; + var errors = r.Errors; // !null & count > 0 + var error = errors[0] as UnexpectedTokenSyntaxError; // + var token = error.UnexpectedToken.TokenID; // comma + var line = error.Line; // 3 + var column = error.Column; // 12 - //ParserBuilder builder = new ParserBuilder(); - //Parser parser = builder.BuildParser(new EbnfJsonParser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); - //Lexer lexer = (Lexer)LexerBuilder.BuildLexer(); - //Stopwatch sw = new Stopwatch(); - - //sw.Start(); - //string json = File.ReadAllText("test.json"); - //var result = lexer.Tokenize(json).ToList(); - //sw.Stop(); - //long milli = sw.ElapsedMilliseconds; - //Console.WriteLine($"wo/ optim : {milli} ms"); - //sw.Reset(); - //sw.Start(); - //json = File.ReadAllText("test.json"); - //result = lexer.Tokenize(json).ToList(); - //sw.Stop(); - //milli = sw.ElapsedMilliseconds; - //Console.WriteLine($"w/ optim : {milli} ms"); - //TestFactorial(); - //testLexerBuilder(); - - - //testJSONLexer(); - //testGenericLexerJSON(); - - - // TestSingleComment(); - //testGenericLexerJson(); + + } + + static void Main(string[] args) + { + testErrors(); + Console.WriteLine("so what ?"); ; diff --git a/sly/lexer/Token.cs b/sly/lexer/Token.cs index ac67dab8..769641d7 100644 --- a/sly/lexer/Token.cs +++ b/sly/lexer/Token.cs @@ -56,6 +56,7 @@ public Token() { End = true; TokenID = DefaultToken; + Position = new TokenPosition(0, 0, 0); } public static Token Empty() { diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index dec62fd7..f376c600 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -157,6 +157,7 @@ protected virtual ParserConfiguration ExtractParserConfiguration(Type p Rule r = BuildNonTerminal(ntAndRule); r.SetVisitor(m); + r.NonTerminalName = ntAndRule.Item1; string key = ntAndRule.Item1 + "__" + r.Key; functions[key] = m; NonTerminal nonT = null; diff --git a/sly/parser/parser/UnexpectedTokenSyntaxError.cs b/sly/parser/parser/UnexpectedTokenSyntaxError.cs index 34622bce..9d952ab5 100644 --- a/sly/parser/parser/UnexpectedTokenSyntaxError.cs +++ b/sly/parser/parser/UnexpectedTokenSyntaxError.cs @@ -17,14 +17,14 @@ public class UnexpectedTokenSyntaxError : ParseError, IComparable public override int Line { get { int? l = UnexpectedToken?.Position?.Line; - return l.HasValue ? l.Value : 0; + return l.HasValue ? l.Value : 1; } } public override int Column { get { int? c = UnexpectedToken?.Position?.Column; - return c.HasValue ? c.Value : 0; + return c.HasValue ? c.Value : 1; } } @@ -51,7 +51,7 @@ public override string ErrorMessage } public UnexpectedTokenSyntaxError(Token unexpectedToken, params T[] expectedTokens) - { + { this.UnexpectedToken = unexpectedToken; if (expectedTokens != null) { diff --git a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs index b24c652a..b7751473 100644 --- a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs @@ -129,30 +129,7 @@ private void InitStartingTokensWithOption(Rule rule, OptionClause option public override SyntaxParseResult Parse(IList> tokens, Rule rule, int position, string nonTerminalName) { - if (rule.NonTerminalName == "field") - { - - ; - } - if (rule.NonTerminalName == "struct") - { - - ; - - } - if (rule.NonTerminalName == "fields") - { - - ; - - } - if (rule.NonTerminalName == "nextFields") - { - - ; - - } int currentPosition = position; List> errors = new List>(); diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs index de0f5815..863d49e0 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs @@ -297,7 +297,6 @@ public SyntaxParseResult ParseTerminal(IList> tokens, TerminalClau public SyntaxParseResult ParseNonTerminal(IList> tokens, NonTerminalClause nonTermClause, int currentPosition) { - NonTerminal nt = Configuration.NonTerminals[nonTermClause.NonTerminalName]; bool found = false; bool isError = false; @@ -312,10 +311,6 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, NonTermin { allAcceptableTokens.AddRange(r.PossibleLeadingTokens); } - else - { - ; - } }); allAcceptableTokens = allAcceptableTokens.Distinct().ToList(); @@ -346,12 +341,19 @@ public SyntaxParseResult ParseNonTerminal(IList> tokens, NonTermin } bool other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0; if ((innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null && - innerRuleRes.Errors.Any()) || other) + !innerRuleRes.Errors.Any()) || other) { greaterIndex = innerRuleRes.EndingPosition; innerRuleErrors.Clear(); + if (innerRuleRes.Errors.Count == 1 && innerRuleRes.Errors[0].Line == 1 && innerRuleRes.Errors[0].Column == 1) + { + var err = innerRuleRes.Errors[0]; + var expect = err.ExpectedTokens; + ; + } innerRuleErrors.AddRange(innerRuleRes.Errors); } + innerRuleErrors.AddRange(innerRuleRes.Errors); allRulesInError = allRulesInError && innerRuleRes.IsError; i++; }