From 3f7cf3cbb11a8cbc4cda45845c1667d0a7185103 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 17 Dec 2019 11:19:40 +0100 Subject: [PATCH] bugfix : - reset lexer before each parse to avoid position shifting - better error reporting on parser build (when parsing production rules) --- samples/ParserExample/Program.cs | 97 +++++++++++++++++------ samples/jsonparser/JSONLexer.cs | 6 +- sly/lexer/GenericLexer.cs | 5 ++ sly/lexer/ILexer.cs | 2 + sly/lexer/Lexer.cs | 5 ++ sly/lexer/fsm/FSMLexer.cs | 7 ++ sly/parser/generator/EBNFParserBuilder.cs | 3 +- sly/parser/parser/Parser.cs | 2 +- sly/sly.csproj | 4 +- 9 files changed, 99 insertions(+), 32 deletions(-) diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index da768b64..0bec7f84 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -405,34 +405,33 @@ public static void test104() public static void testJSON() { try { - Console.WriteLine("starting json test."); var instance = new EbnfJsonGenericParser(); var builder = new ParserBuilder(); var buildResult = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); - if (buildResult.IsOk) - { - Console.WriteLine("parser built."); - var parser = buildResult.Result; - var content = File.ReadAllText("test.json"); - Console.WriteLine("test.json read."); - var jsonResult = parser.Parse(content); - Console.WriteLine("json parse done."); - if (jsonResult.IsOk) - { - Console.WriteLine("YES !"); - } - else - { - Console.WriteLine("Ooh no !"); - } - Console.WriteLine("Done."); - - } - else - { - buildResult.Errors.ForEach(e => Console.WriteLine(e.Message)); - } + // if (buildResult.IsOk) + // { + // Console.WriteLine("parser built."); + // var parser = buildResult.Result; + // var content = File.ReadAllText("test.json"); + // Console.WriteLine("test.json read."); + // var jsonResult = parser.Parse(content); + // Console.WriteLine("json parse done."); + // if (jsonResult.IsOk) + // { + // Console.WriteLine("YES !"); + // } + // else + // { + // Console.WriteLine("Ooh no !"); + // } + // Console.WriteLine("Done."); + // + // } + // else + // { + // buildResult.Errors.ForEach(e => Console.WriteLine(e.Message)); + // } } catch(Exception e) { Console.WriteLine($"ERROR {e.Message} : \n {e.StackTrace}"); @@ -494,15 +493,61 @@ private static void TestChars() } } + private static void TestGrammarParser() + { + string productionRule = "clauses : clause (COMMA [D] clause)*"; + var ruleparser = new RuleParser(); + var builder = new ParserBuilder>(); + var grammarParser = builder.BuildParser(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule").Result; + var result = grammarParser.Parse(productionRule); + grammarParser.Lexer.ResetLexer(); + //(grammarParser.Lexer as GenericLexer).ResetLexer(); + Console.WriteLine($"alors ? {string.Join('\n',result.Errors.Select(e => e.ErrorMessage))}"); + result = grammarParser.Parse(productionRule); + Console.WriteLine($"alors ? {string.Join('\n',result.Errors.Select(e => e.ErrorMessage))}"); + ; + + Console.WriteLine("starting"); + ErroneousGrammar parserInstance = new ErroneousGrammar(); + Console.WriteLine("new instance"); + + var builder2 = new ParserBuilder(); + Console.WriteLine("builder"); + + var Parser = builder.BuildParser(parserInstance,ParserType.EBNF_LL_RECURSIVE_DESCENT,"rule"); + Console.WriteLine($"built : {Parser.IsOk}"); + + + } + private static void Main(string[] args) { //TestContextualParser(); //TestTokenCallBacks(); //test104(); //testJSON(); - TestGraphViz(); + TestGrammarParser(); + // TestGraphViz(); //TestGraphViz(); - TestChars(); + //TestChars(); + } } + + public enum TestGrammarToken + { + [Lexeme(GenericToken.SugarToken,",")] + COMMA = 1 + } + + public class ErroneousGrammar + { + [Production("clauses : clause (COMMA [D] clause)*")] + + public object test() + { + return null; + } + } + } \ No newline at end of file diff --git a/samples/jsonparser/JSONLexer.cs b/samples/jsonparser/JSONLexer.cs index 961fdae6..d3f3884c 100644 --- a/samples/jsonparser/JSONLexer.cs +++ b/samples/jsonparser/JSONLexer.cs @@ -202,9 +202,11 @@ public LexerResult Tokenize(ReadOnlyMemory source) position++; } } - - return new LexerResult(tokens); } + + public void ResetLexer() + { + } } } \ No newline at end of file diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs index c384d5e4..ad3fc293 100644 --- a/sly/lexer/GenericLexer.cs +++ b/sly/lexer/GenericLexer.cs @@ -752,5 +752,10 @@ public string ToGraphViz() { return LexerFsm.ToGraphViz(); } + + public void ResetLexer() + { + LexerFsm.Reset(); + } } } \ No newline at end of file diff --git a/sly/lexer/ILexer.cs b/sly/lexer/ILexer.cs index 54964e7f..4f90e316 100644 --- a/sly/lexer/ILexer.cs +++ b/sly/lexer/ILexer.cs @@ -9,5 +9,7 @@ public interface ILexer where T : struct LexerResult Tokenize(string source); LexerResult Tokenize(ReadOnlyMemory source); + + void ResetLexer(); } } \ No newline at end of file diff --git a/sly/lexer/Lexer.cs b/sly/lexer/Lexer.cs index d50cb276..0f0fa7d6 100644 --- a/sly/lexer/Lexer.cs +++ b/sly/lexer/Lexer.cs @@ -82,5 +82,10 @@ public LexerResult Tokenize(ReadOnlyMemory source) { throw new NotImplementedException(); } + + public void ResetLexer() + { + + } } } \ No newline at end of file diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs index 823ab94f..b69b24bd 100644 --- a/sly/lexer/fsm/FSMLexer.cs +++ b/sly/lexer/fsm/FSMLexer.cs @@ -269,5 +269,12 @@ private void ConsumeIgnored(ReadOnlyMemory source) } #endregion + + public void Reset() + { + CurrentColumn = 0; + CurrentLine = 0; + CurrentPosition = 0; + } } } \ No newline at end of file diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 8633d1ea..83a1c218 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -34,7 +34,7 @@ public override BuildResult> BuildParser(object parserInstance, } catch (Exception e) { - result.AddError(new ParserInitializationError(ErrorLevel.ERROR, e.Message+"\n"+e.StackTrace)); + result.AddError(new ParserInitializationError(ErrorLevel.ERROR, e.Message)); return result; } @@ -128,6 +128,7 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty .Errors .Select(e => e.ErrorMessage) .Aggregate((e1, e2) => e1 + "\n" + e2); + message = $"rule error [{ruleString}] : {message}"; throw new ParserConfigurationException(message); } } diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs index 3db6ca3d..b20447ed 100644 --- a/sly/parser/parser/Parser.cs +++ b/sly/parser/parser/Parser.cs @@ -58,7 +58,7 @@ public ParseResult Parse(string source, string startingNonTerminal = nu public ParseResult ParseWithContext(string source, object context, string startingNonTerminal = null) { ParseResult result = null; - + Lexer.ResetLexer(); var lexingResult = Lexer.Tokenize(source); if (lexingResult.IsError) { diff --git a/sly/sly.csproj b/sly/sly.csproj index c7ed74de..c86a5f70 100644 --- a/sly/sly.csproj +++ b/sly/sly.csproj @@ -6,11 +6,11 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 2.4.1.0 + 2.4.1.1 https://github.com/b3b00/sly https://github.com/b3b00/sly https://github.com/b3b00/sly/blob/master/LICENSE - 2.4.1.0 + 2.4.1.1 1701;1702;1705;1591