Skip to content

Commit

Permalink
Merge pull request #63 from b3b00/bugfix/genericLexerErrors
Browse files Browse the repository at this point in the history 
generic lexer error reporting (build and lex time)
  • Loading branch information
@b3b00
b3b00 authored Mar 6, 2018
2 parents 416b785 + 9f32017 commit 6b60cd5
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 44 deletions.
2 changes: 1 addition & 1 deletion ParserTests/CommentsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public void TestInnerMultiComment() {
Assert.Equal(CommentsToken.INT,token4.TokenID);
Assert.Equal("3",token4.Value);
Assert.Equal(1,token4.Position.Line);
Assert.Equal(13,token4.Position.Column);
Assert.Equal(14,token4.Position.Column);

Assert.Equal(CommentsToken.INT,token5.TokenID);
Assert.Equal("4",token5.Value);
Expand Down
140 changes: 103 additions & 37 deletions ParserTests/GenericLexerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,58 +7,66 @@
using System.Linq;
using System.Collections.Generic;
using System.Text;
using System;
using Xunit;
using sly.buildresult;
using sly.lexer.fsm;

namespace ParserTests
{

public enum Extensions {
[Lexeme(GenericToken.Extension)]
DATE,
public enum Extensions
{
[Lexeme(GenericToken.Extension)]
DATE,

[Lexeme(GenericToken.Double)]
DOUBLE,
[Lexeme(GenericToken.Double)]
DOUBLE,
}


public static class ExtendedGenericLexer


public static class ExtendedGenericLexer
{



public static bool CheckDate(string value) {

public static bool CheckDate(string value)
{
bool ok = false;
if (value.Length==5) {
if (value.Length == 5)
{
ok = char.IsDigit(value[0]);
ok = ok && char.IsDigit(value[1]);
ok = ok && value[2] == '.';
ok = ok && char.IsDigit(value[3]);
ok = ok && char.IsDigit(value[4]);
}
}
return ok;
}



public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer<Extensions> lexer) {
if (token == Extensions.DATE) {
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer<Extensions> lexer)
{
if (token == Extensions.DATE)
{



NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
{
match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
return match;
match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
return match;
};

var fsmBuilder = lexer.FSMBuilder;

// TODO
fsmBuilder.GoTo(GenericLexer<Extensions>.in_double)
.Transition('.',CheckDate)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4,"[0-9]")
.RepetitionTransition(4, "[0-9]")
// .RangeTransition('0','9')
// .Mark("y1")
// .RangeTransition('0','9')
Expand All @@ -74,33 +82,48 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic

}

public enum DoubleQuotedString {
public enum BadLetterStringDelimiter
{
[Lexeme(GenericToken.String, "a")]
Letter
}

[Lexeme(GenericToken.String,"\"")]
public enum BadEmptyStringDelimiter
{
[Lexeme(GenericToken.String, "")]
Empty
}

public enum DoubleQuotedString
{

[Lexeme(GenericToken.String, "\"")]
DoubleString
}

public enum SingleQuotedString {
public enum SingleQuotedString
{

[Lexeme(GenericToken.String,"'")]
[Lexeme(GenericToken.String, "'")]
SingleString
}

public enum DefaultQuotedString {
public enum DefaultQuotedString
{

[Lexeme(GenericToken.String)]
[Lexeme(GenericToken.String)]
DefaultString
}

public enum AlphaId
{
[Lexeme(GenericToken.Identifier,IdentifierType.Alpha)]
[Lexeme(GenericToken.Identifier, IdentifierType.Alpha)]
ID
}

public enum AlphaNumId
{
[Lexeme(GenericToken.Identifier,IdentifierType.AlphaNumeric)]
[Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)]
ID
}

Expand All @@ -115,19 +138,19 @@ public class GenericLexerTests


[Fact]
public void TestExtensions()
public void TestExtensions()
{
var lexerRes = LexerBuilder.BuildLexer<Extensions>(new BuildResult<ILexer<Extensions>>(), ExtendedGenericLexer.AddExtension);
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer<Extensions>;
Assert.NotNull(lexer);

List<Token<Extensions>> tokens = lexer.Tokenize("20.02.2018 3.14").ToList();
Assert.Equal(2,tokens.Count);
Assert.Equal(Extensions.DATE,tokens[0].TokenID);
Assert.Equal("20.02.2018",tokens[0].Value);
Assert.Equal(Extensions.DOUBLE,tokens[1].TokenID);
Assert.Equal("3.14",tokens[1].Value);
Assert.Equal(2, tokens.Count);
Assert.Equal(Extensions.DATE, tokens[0].TokenID);
Assert.Equal("20.02.2018", tokens[0].Value);
Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID);
Assert.Equal("3.14", tokens[1].Value);
}

[Fact]
Expand Down Expand Up @@ -189,7 +212,8 @@ public void TestAlphaNumDashIdStartsWithUnderscore()
}

[Fact]
public void TestDoubleQuotedString() {
public void TestDoubleQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<DoubleQuotedString>(new BuildResult<ILexer<DoubleQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -201,8 +225,9 @@ public void TestDoubleQuotedString() {
Assert.Equal(source, tok.StringWithoutQuotes);
}

[Fact]
public void TestSingleQuotedString() {
[Fact]
public void TestSingleQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<SingleQuotedString>(new BuildResult<ILexer<SingleQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -215,7 +240,8 @@ public void TestSingleQuotedString() {
}

[Fact]
public void TestDefaultQuotedString() {
public void TestDefaultQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<DefaultQuotedString>(new BuildResult<ILexer<DefaultQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -225,6 +251,46 @@ public void TestDefaultQuotedString() {
Token<DefaultQuotedString> tok = r[0];
Assert.Equal(DefaultQuotedString.DefaultString, tok.TokenID);
Assert.Equal(source, tok.StringWithoutQuotes);
}

[Fact]
public void TestBadLetterStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer<BadLetterStringDelimiter>(new BuildResult<ILexer<BadLetterStringDelimiter>>());
Assert.True(lexerRes.IsError);
Assert.Equal(1, lexerRes.Errors.Count);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.True(error.Message.Contains("can not start with a letter"));
}

[Fact]
public void TestBadEmptyStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer<BadEmptyStringDelimiter>(new BuildResult<ILexer<BadEmptyStringDelimiter>>());
Assert.True(lexerRes.IsError);
Assert.Equal(1, lexerRes.Errors.Count);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.True(error.Message.Contains("must be 1 character length"));
}

[Fact]
public void TestLexerError()
{
var lexerRes = LexerBuilder.BuildLexer<AlphaNumDashId>(new BuildResult<ILexer<AlphaNumDashId>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
string source = "hello world 2 + 2 ";
var errException = Assert.Throws<LexerException<GenericToken>>(() => lexer.Tokenize(source).ToList());
var error = errException.Error;
Assert.Equal(0, error.Line);
Assert.Equal(13, error.Column);
Assert.Equal('2', error.UnexpectedChar);




}
}
}
9 changes: 4 additions & 5 deletions sly/lexer/GenericLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,11 +291,11 @@ public void AddStringLexem(IN token, string stringDelimiter)
{
if (string.IsNullOrEmpty(stringDelimiter) || stringDelimiter.Length > 1)
{
throw new ArgumentException($"bad lexem {stringDelimiter} : StringToken lexeme <{token.ToString()}> must be 1 character length.");
throw new InvalidLexerException($"bad lexem {stringDelimiter} : StringToken lexeme <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(stringDelimiter[0]))
{
throw new ArgumentException($"bad lexem {stringDelimiter} : SugarToken lexeme <{token.ToString()}> can not start with a letter.");
throw new InvalidLexerException($"bad lexem {stringDelimiter} : SugarToken lexeme <{token.ToString()}> can not start with a letter.");
}

StringDelimiter = stringDelimiter[0];
Expand Down Expand Up @@ -327,7 +327,7 @@ public void AddSugarLexem(IN token, string specialValue)
{
if (char.IsLetter(specialValue[0]))
{
throw new ArgumentException($"bad lexem {specialValue} : SugarToken lexeme <{token.ToString()}> can not start with a letter.");
throw new InvalidLexerException($"bad lexem {specialValue} : SugarToken lexeme <{token.ToString()}> can not start with a letter.");
}
NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
{
Expand Down Expand Up @@ -411,8 +411,7 @@ public void ConsumeComment(Token<GenericToken> comment, string source)
}
else
{
int t = LexerFsm.CurrentColumn+lines[0].Length-MultiLineCommentEnd.Length;
newColumn = LexerFsm.CurrentColumn+lines[0].Length;
newColumn = LexerFsm.CurrentColumn+lines[0].Length+MultiLineCommentEnd.Length;
}


Expand Down
10 changes: 10 additions & 0 deletions sly/lexer/InvalidLexerException.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using System;

namespace sly.lexer {
public class InvalidLexerException : Exception
{
public InvalidLexerException(string message) : base(message)
{
}
}
}
6 changes: 6 additions & 0 deletions sly/lexer/LexerBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,13 @@ private static BuildResult<ILexer<IN>> BuildGenericLexer<IN>(Dictionary<IN, List
{
if (lexem.GenericTokenParameters != null && lexem.GenericTokenParameters.Length > 0)
{
try {
lexer.AddStringLexem(tokenID, lexem.GenericTokenParameters[0]);
}
catch (Exception e) {
result.IsError = true;
result.AddError(new InitializationError(ErrorLevel.FATAL,e.Message));
}
}
else
{
Expand Down
9 changes: 8 additions & 1 deletion sly/lexer/fsm/FSMLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,14 @@ public FSMMatch<N> Run(string source, int start)
successes.Push(resultInter);
}
CurrentPosition++;
CurrentColumn += value.Length;
CurrentColumn++ ;
}
else {

if (lastNode == 0 && !tokenStarted && !successes.Any() && CurrentPosition < source.Length) {
throw new LexerException<T>(new LexicalError(CurrentLine,CurrentColumn, source[CurrentPosition]));
}
;
}

}
Expand Down

0 comments on commit 6b60cd5

Please sign in to comment.