Skip to content

Commit

Permalink
Merge pull request #64 from b3b00/dev
Browse files Browse the repository at this point in the history 
2.0.6.2
  • Loading branch information
@b3b00
b3b00 authored Mar 6, 2018
2 parents 7060f90 + 99847ed commit a4aaa8e
Show file tree
Hide file tree
Showing 8 changed files with 253 additions and 90 deletions.
95 changes: 92 additions & 3 deletions ParserTests/CommentsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,6 @@ public void TestGenericMultiLineComment()

var tokens = lexer.Tokenize(code).ToList();




Assert.Equal(4, tokens.Count);

var token1 = tokens[0];
Expand Down Expand Up @@ -127,6 +124,98 @@ public void TestGenericMultiLineComment()

}

[Fact]
public void TestInnerMultiComment() {
var lexerRes = LexerBuilder.BuildLexer<CommentsToken>(new BuildResult<ILexer<CommentsToken>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer<CommentsToken>;


string dump = lexer.ToString();

string code = @"1
2 /* inner */ 3
4
";

var tokens = lexer.Tokenize(code).ToList();

Assert.Equal(5, tokens.Count);

var token1 = tokens[0];
var token2 = tokens[1];
var token3 = tokens[2];
var token4 = tokens[3];
var token5 = tokens[4];


Assert.Equal(CommentsToken.INT,token1.TokenID);
Assert.Equal("1",token1.Value);
Assert.Equal(0,token1.Position.Line);
Assert.Equal(0,token1.Position.Column);

Assert.Equal(CommentsToken.INT,token2.TokenID);
Assert.Equal("2",token2.Value);
Assert.Equal(1,token2.Position.Line);
Assert.Equal(0,token2.Position.Column);

Assert.Equal(CommentsToken.COMMENT,token3.TokenID);
Assert.Equal(@" inner ",token3.Value);
Assert.Equal(1,token3.Position.Line);
Assert.Equal(2,token3.Position.Column);

Assert.Equal(CommentsToken.INT,token4.TokenID);
Assert.Equal("3",token4.Value);
Assert.Equal(1,token4.Position.Line);
Assert.Equal(14,token4.Position.Column);

Assert.Equal(CommentsToken.INT,token5.TokenID);
Assert.Equal("4",token5.Value);
Assert.Equal(2,token5.Position.Line);
Assert.Equal(0,token5.Position.Column);
}

[Fact]
public void NotEndingMultiComment() {
var lexerRes = LexerBuilder.BuildLexer<CommentsToken>(new BuildResult<ILexer<CommentsToken>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer<CommentsToken>;


string dump = lexer.ToString();

string code = @"1
2 /* not ending
comment";

var tokens = lexer.Tokenize(code).ToList();

Assert.Equal(3, tokens.Count);

var token1 = tokens[0];
var token2 = tokens[1];
var token3 = tokens[2];


Assert.Equal(CommentsToken.INT,token1.TokenID);
Assert.Equal("1",token1.Value);
Assert.Equal(0,token1.Position.Line);
Assert.Equal(0,token1.Position.Column);

Assert.Equal(CommentsToken.INT,token2.TokenID);
Assert.Equal("2",token2.Value);
Assert.Equal(1,token2.Position.Line);
Assert.Equal(0,token2.Position.Column);

Assert.Equal(CommentsToken.COMMENT,token3.TokenID);
Assert.Equal(@" not ending
comment",token3.Value);
Assert.Equal(1,token3.Position.Line);
Assert.Equal(2,token3.Position.Column);


}



}
Expand Down
140 changes: 103 additions & 37 deletions ParserTests/GenericLexerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,58 +7,66 @@
using System.Linq;
using System.Collections.Generic;
using System.Text;
using System;
using Xunit;
using sly.buildresult;
using sly.lexer.fsm;

namespace ParserTests
{

public enum Extensions {
[Lexeme(GenericToken.Extension)]
DATE,
public enum Extensions
{
[Lexeme(GenericToken.Extension)]
DATE,

[Lexeme(GenericToken.Double)]
DOUBLE,
[Lexeme(GenericToken.Double)]
DOUBLE,
}


public static class ExtendedGenericLexer


public static class ExtendedGenericLexer
{



public static bool CheckDate(string value) {

public static bool CheckDate(string value)
{
bool ok = false;
if (value.Length==5) {
if (value.Length == 5)
{
ok = char.IsDigit(value[0]);
ok = ok && char.IsDigit(value[1]);
ok = ok && value[2] == '.';
ok = ok && char.IsDigit(value[3]);
ok = ok && char.IsDigit(value[4]);
}
}
return ok;
}



public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer<Extensions> lexer) {
if (token == Extensions.DATE) {
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer<Extensions> lexer)
{
if (token == Extensions.DATE)
{



NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
{
match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
return match;
match.Properties[GenericLexer<Extensions>.DerivedToken] = Extensions.DATE;
return match;
};

var fsmBuilder = lexer.FSMBuilder;

// TODO
fsmBuilder.GoTo(GenericLexer<Extensions>.in_double)
.Transition('.',CheckDate)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4,"[0-9]")
.RepetitionTransition(4, "[0-9]")
// .RangeTransition('0','9')
// .Mark("y1")
// .RangeTransition('0','9')
Expand All @@ -74,33 +82,48 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic

}

public enum DoubleQuotedString {
public enum BadLetterStringDelimiter
{
[Lexeme(GenericToken.String, "a")]
Letter
}

[Lexeme(GenericToken.String,"\"")]
public enum BadEmptyStringDelimiter
{
[Lexeme(GenericToken.String, "")]
Empty
}

public enum DoubleQuotedString
{

[Lexeme(GenericToken.String, "\"")]
DoubleString
}

public enum SingleQuotedString {
public enum SingleQuotedString
{

[Lexeme(GenericToken.String,"'")]
[Lexeme(GenericToken.String, "'")]
SingleString
}

public enum DefaultQuotedString {
public enum DefaultQuotedString
{

[Lexeme(GenericToken.String)]
[Lexeme(GenericToken.String)]
DefaultString
}

public enum AlphaId
{
[Lexeme(GenericToken.Identifier,IdentifierType.Alpha)]
[Lexeme(GenericToken.Identifier, IdentifierType.Alpha)]
ID
}

public enum AlphaNumId
{
[Lexeme(GenericToken.Identifier,IdentifierType.AlphaNumeric)]
[Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)]
ID
}

Expand All @@ -115,19 +138,19 @@ public class GenericLexerTests


[Fact]
public void TestExtensions()
public void TestExtensions()
{
var lexerRes = LexerBuilder.BuildLexer<Extensions>(new BuildResult<ILexer<Extensions>>(), ExtendedGenericLexer.AddExtension);
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer<Extensions>;
Assert.NotNull(lexer);

List<Token<Extensions>> tokens = lexer.Tokenize("20.02.2018 3.14").ToList();
Assert.Equal(2,tokens.Count);
Assert.Equal(Extensions.DATE,tokens[0].TokenID);
Assert.Equal("20.02.2018",tokens[0].Value);
Assert.Equal(Extensions.DOUBLE,tokens[1].TokenID);
Assert.Equal("3.14",tokens[1].Value);
Assert.Equal(2, tokens.Count);
Assert.Equal(Extensions.DATE, tokens[0].TokenID);
Assert.Equal("20.02.2018", tokens[0].Value);
Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID);
Assert.Equal("3.14", tokens[1].Value);
}

[Fact]
Expand Down Expand Up @@ -189,7 +212,8 @@ public void TestAlphaNumDashIdStartsWithUnderscore()
}

[Fact]
public void TestDoubleQuotedString() {
public void TestDoubleQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<DoubleQuotedString>(new BuildResult<ILexer<DoubleQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -201,8 +225,9 @@ public void TestDoubleQuotedString() {
Assert.Equal(source, tok.StringWithoutQuotes);
}

[Fact]
public void TestSingleQuotedString() {
[Fact]
public void TestSingleQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<SingleQuotedString>(new BuildResult<ILexer<SingleQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -215,7 +240,8 @@ public void TestSingleQuotedString() {
}

[Fact]
public void TestDefaultQuotedString() {
public void TestDefaultQuotedString()
{
var lexerRes = LexerBuilder.BuildLexer<DefaultQuotedString>(new BuildResult<ILexer<DefaultQuotedString>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
Expand All @@ -225,6 +251,46 @@ public void TestDefaultQuotedString() {
Token<DefaultQuotedString> tok = r[0];
Assert.Equal(DefaultQuotedString.DefaultString, tok.TokenID);
Assert.Equal(source, tok.StringWithoutQuotes);
}

[Fact]
public void TestBadLetterStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer<BadLetterStringDelimiter>(new BuildResult<ILexer<BadLetterStringDelimiter>>());
Assert.True(lexerRes.IsError);
Assert.Equal(1, lexerRes.Errors.Count);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.True(error.Message.Contains("can not start with a letter"));
}

[Fact]
public void TestBadEmptyStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer<BadEmptyStringDelimiter>(new BuildResult<ILexer<BadEmptyStringDelimiter>>());
Assert.True(lexerRes.IsError);
Assert.Equal(1, lexerRes.Errors.Count);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.True(error.Message.Contains("must be 1 character length"));
}

[Fact]
public void TestLexerError()
{
var lexerRes = LexerBuilder.BuildLexer<AlphaNumDashId>(new BuildResult<ILexer<AlphaNumDashId>>());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
string source = "hello world 2 + 2 ";
var errException = Assert.Throws<LexerException<GenericToken>>(() => lexer.Tokenize(source).ToList());
var error = errException.Error;
Assert.Equal(0, error.Line);
Assert.Equal(13, error.Column);
Assert.Equal('2', error.UnexpectedChar);




}
}
}
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
# C# Lex Yacc #

[![Build status](https://ci.appveyor.com/api/projects/status/n9uffgkqn2qet7k9?svg=true)](https://ci.appveyor.com/project/OlivierDuhart/sly)
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/b3b00/sly/blob/dev/LICENSE)
[![NuGet](https://img.shields.io/nuget/v/sly.svg)](https://www.nuget.org/packages/sly/)
![AppVeyor tests](https://img.shields.io/appveyor/tests/OlivierDuhart/sly.svg)
[![codecov](https://codecov.io/gh/b3b00/csly/branch/dev/graph/badge.svg)](https://codecov.io/gh/b3b00/csly)


[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/b3b00/sly/blob/dev/LICENSE)
![NuGet](https://img.shields.io/nuget/v/sly.svg)


:warning: This readme is a bit out of date. Go to to the [wiki](https://github.com/b3b00/csly/wiki) for a more up to date documentation.

#LY is a parser generator halfway between parser combinators and parser generator like
Expand Down
Loading

0 comments on commit a4aaa8e

Please sign in to comment.