Skip to content

Commit

Permalink
error reporting bugfix
Browse files Browse the repository at this point in the history
  • Loading branch information
@olduh29
olduh29 committed May 21, 2018
1 parent 310b6e3 commit 2c0bb2e
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 70 deletions.
37 changes: 35 additions & 2 deletions ParserTests/EBNFTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.Linq;
using Xunit;
using System.Text;
using sly.parser;
using sly.lexer;
using sly.parser.generator;
Expand Down Expand Up @@ -45,11 +46,32 @@ public string root(Token<OptionTestToken> a, string b, Token<OptionTestToken> c)
else {
r = $"{r},{c.Value})";
}

return r;
}

[Production("B : b ")]
[Production("root : a b? c ")]
public string root2(Token<OptionTestToken> a, Token<OptionTestToken> b, Token<OptionTestToken> c)
{
StringBuilder result = new StringBuilder();
result.Append("R(");
result.Append(a.StringWithoutQuotes);
result.Append(",");
if (b.IsEmpty)
{
result.Append("<none>");
}
else
{
result.Append(b.StringWithoutQuotes);
}
result.Append(",");
result.Append(c.StringWithoutQuotes);
result.Append(")");

return result.ToString();
}

[Production("B : b ")]
public string bee(Token<OptionTestToken> b) {
return $"B({b.Value})";
}
Expand Down Expand Up @@ -321,6 +343,17 @@ public void TestEmptyOption() {
Assert.Equal("R(a,B(b),<none>)", result.Result);
}

[Fact]
public void TestEmptyOptionInMiddle()
{
var buildResult = BuildOptionParser();
Assert.False(buildResult.IsError);
var optionParser = buildResult.Result;

var result = optionParser.Parse("a c");
Assert.Equal("R(a,<none>,c)", result.Result);
}

private void AssertString(JObject obj, string key, string value)
{
Assert.True(obj.ContainsKey(key));
Expand Down
14 changes: 7 additions & 7 deletions ParserTests/ErrorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ public void TestJsonSyntaxError()


string source = @"{
'one': 1,
'bug':{,}
}".Replace("'", "\"");
'one': 1,
'bug':{,}
}".Replace("'", "\"");
ParseResult<JsonToken,JSon> r = parser.Parse(source);
Assert.True(r.IsError);
Assert.Null(r.Result);
Expand All @@ -33,8 +33,8 @@ public void TestJsonSyntaxError()
UnexpectedTokenSyntaxError<JsonToken> error = r.Errors[0] as UnexpectedTokenSyntaxError<JsonToken>;

Assert.Equal(JsonToken.COMMA, error?.UnexpectedToken.TokenID);
Assert.Equal(3, error?.Line);
Assert.Equal(24, error?.Column);
Assert.Equal(2, error?.Line);
Assert.Equal(13, error?.Column);

}

Expand All @@ -49,13 +49,13 @@ public void TestExpressionSyntaxError()
Assert.True(r.IsError);
Assert.NotNull(r.Errors);
Assert.True(r.Errors.Count > 0);
Assert.IsAssignableFrom(typeof(UnexpectedTokenSyntaxError<ExpressionToken>), r.Errors[0]);
Assert.IsAssignableFrom < UnexpectedTokenSyntaxError<ExpressionToken>>(r.Errors[0]);
UnexpectedTokenSyntaxError<ExpressionToken> error = r.Errors[0] as UnexpectedTokenSyntaxError<ExpressionToken>;

Assert.Equal(ExpressionToken.PLUS, error.UnexpectedToken.TokenID);

Assert.Equal(1, error.Line);
Assert.Equal(10, error.Column);
Assert.Equal(8, error.Column);
}

[Fact]
Expand Down
56 changes: 27 additions & 29 deletions samples/ParserExample/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,37 +331,35 @@ static void testJSONLexer()
;
}

static void Main(string[] args)

static void testErrors()
{
JSONParser jsonParser = new JSONParser();
ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
Parser<JsonToken, JSon> parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root").Result;


string source = @"{
'one': 1,
'bug':{,}
}".Replace("'", "\"");
ParseResult<JsonToken, JSon> r = parser.Parse(source);

bool isError = r.IsError; // true
var root = r.Result; // null;
var errors = r.Errors; // !null & count > 0
var error = errors[0] as UnexpectedTokenSyntaxError<JsonToken>; //
var token = error.UnexpectedToken.TokenID; // comma
var line = error.Line; // 3
var column = error.Column; // 12

//ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
//Parser<JsonToken, JSon> parser = builder.BuildParser(new EbnfJsonParser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
//Lexer<JsonToken> lexer = (Lexer<JsonToken>)LexerBuilder.BuildLexer<JsonToken>();
//Stopwatch sw = new Stopwatch();

//sw.Start();
//string json = File.ReadAllText("test.json");
//var result = lexer.Tokenize(json).ToList();
//sw.Stop();
//long milli = sw.ElapsedMilliseconds;
//Console.WriteLine($"wo/ optim : {milli} ms");
//sw.Reset();
//sw.Start();
//json = File.ReadAllText("test.json");
//result = lexer.Tokenize(json).ToList();
//sw.Stop();
//milli = sw.ElapsedMilliseconds;
//Console.WriteLine($"w/ optim : {milli} ms");
//TestFactorial();
//testLexerBuilder();


//testJSONLexer();
//testGenericLexerJSON();


// TestSingleComment();
//testGenericLexerJson();

}

static void Main(string[] args)
{
testErrors();

Console.WriteLine("so what ?");

;
Expand Down
1 change: 1 addition & 0 deletions sly/lexer/Token.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public Token()
{
End = true;
TokenID = DefaultToken;
Position = new TokenPosition(0, 0, 0);
}

public static Token<T> Empty() {
Expand Down
1 change: 1 addition & 0 deletions sly/parser/generator/ParserBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ protected virtual ParserConfiguration<IN,OUT> ExtractParserConfiguration(Type p

Rule<IN> r = BuildNonTerminal(ntAndRule);
r.SetVisitor(m);
r.NonTerminalName = ntAndRule.Item1;
string key = ntAndRule.Item1 + "__" + r.Key;
functions[key] = m;
NonTerminal<IN> nonT = null;
Expand Down
6 changes: 3 additions & 3 deletions sly/parser/parser/UnexpectedTokenSyntaxError.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ public class UnexpectedTokenSyntaxError<T> : ParseError, IComparable
public override int Line {
get {
int? l = UnexpectedToken?.Position?.Line;
return l.HasValue ? l.Value : 0;
return l.HasValue ? l.Value : 1;
}
}
public override int Column
{
get {
int? c = UnexpectedToken?.Position?.Column;
return c.HasValue ? c.Value : 0;
return c.HasValue ? c.Value : 1;
}
}

Expand All @@ -51,7 +51,7 @@ public override string ErrorMessage
}

public UnexpectedTokenSyntaxError(Token<T> unexpectedToken, params T[] expectedTokens)
{
{
this.UnexpectedToken = unexpectedToken;
if (expectedTokens != null)
{
Expand Down
23 changes: 0 additions & 23 deletions sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,30 +129,7 @@ private void InitStartingTokensWithOption(Rule<IN> rule, OptionClause<IN> option

public override SyntaxParseResult<IN> Parse(IList<Token<IN>> tokens, Rule<IN> rule, int position, string nonTerminalName)
{
if (rule.NonTerminalName == "field")
{

;

}
if (rule.NonTerminalName == "struct")
{

;

}
if (rule.NonTerminalName == "fields")
{

;

}
if (rule.NonTerminalName == "nextFields")
{

;

}

int currentPosition = position;
List<UnexpectedTokenSyntaxError<IN>> errors = new List<UnexpectedTokenSyntaxError<IN>>();
Expand Down
14 changes: 8 additions & 6 deletions sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,6 @@ public SyntaxParseResult<IN> ParseTerminal(IList<Token<IN>> tokens, TerminalClau

public SyntaxParseResult<IN> ParseNonTerminal(IList<Token<IN>> tokens, NonTerminalClause<IN> nonTermClause, int currentPosition)
{

NonTerminal<IN> nt = Configuration.NonTerminals[nonTermClause.NonTerminalName];
bool found = false;
bool isError = false;
Expand All @@ -312,10 +311,6 @@ public SyntaxParseResult<IN> ParseNonTerminal(IList<Token<IN>> tokens, NonTermin
{
allAcceptableTokens.AddRange(r.PossibleLeadingTokens);
}
else
{
;
}
});
allAcceptableTokens = allAcceptableTokens.Distinct<IN>().ToList<IN>();

Expand Down Expand Up @@ -346,12 +341,19 @@ public SyntaxParseResult<IN> ParseNonTerminal(IList<Token<IN>> tokens, NonTermin
}
bool other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0;
if ((innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null &&
innerRuleRes.Errors.Any()) || other)
!innerRuleRes.Errors.Any()) || other)
{
greaterIndex = innerRuleRes.EndingPosition;
innerRuleErrors.Clear();
if (innerRuleRes.Errors.Count == 1 && innerRuleRes.Errors[0].Line == 1 && innerRuleRes.Errors[0].Column == 1)
{
var err = innerRuleRes.Errors[0];
var expect = err.ExpectedTokens;
;
}
innerRuleErrors.AddRange(innerRuleRes.Errors);
}
innerRuleErrors.AddRange(innerRuleRes.Errors);
allRulesInError = allRulesInError && innerRuleRes.IsError;
i++;
}
Expand Down

0 comments on commit 2c0bb2e

Please sign in to comment.