From e752cee841f75a239c87a4b0b8a3bf190c910f93 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 29 Jun 2022 17:14:42 +0200 Subject: [PATCH] simple templating sample using lexer modes --- ParserTests/ParserTests.csproj | 1 + ParserTests/lexer/Issue210Extensions.cs | 2 +- ParserTests/samples/TemplateTests.cs | 39 +++ samples/ParserExample/ParserExample.csproj | 1 + samples/ParserExample/Program.cs | 52 +++- samples/SimpleTemplate/SimpleTemplate.csproj | 11 + samples/SimpleTemplate/TemplateLexer.cs | 161 +++++++++++ samples/SimpleTemplate/TemplateParser.cs | 260 ++++++++++++++++++ samples/XML/MinimalXmlLexer.cs | 2 +- sly.sln | 7 + sly/lexer/GenericLexer.cs | 79 +++++- sly/lexer/LexerBuilder.cs | 12 +- sly/lexer/attributes/AllExcept.cs | 2 +- sly/lexer/fsm/FSMLexerBuilder.cs | 26 +- .../transitioncheck/TransitionAnyExcept.cs | 5 + 15 files changed, 639 insertions(+), 21 deletions(-) create mode 100644 ParserTests/samples/TemplateTests.cs create mode 100644 samples/SimpleTemplate/SimpleTemplate.csproj create mode 100644 samples/SimpleTemplate/TemplateLexer.cs create mode 100644 samples/SimpleTemplate/TemplateParser.cs diff --git a/ParserTests/ParserTests.csproj b/ParserTests/ParserTests.csproj index 73bb7898..8ba13c4b 100644 --- a/ParserTests/ParserTests.csproj +++ b/ParserTests/ParserTests.csproj @@ -27,6 +27,7 @@ + diff --git a/ParserTests/lexer/Issue210Extensions.cs b/ParserTests/lexer/Issue210Extensions.cs index 11a7141f..759bf718 100644 --- a/ParserTests/lexer/Issue210Extensions.cs +++ b/ParserTests/lexer/Issue210Extensions.cs @@ -39,7 +39,7 @@ FSMMatch Callback(FSMMatch match) lexer.FSMBuilder.GoTo(GenericLexer.start) - .Transition('?') + .SafeTransition('?') .Mark("qmark") .ExceptTransition(new[] {'?'}) // moving to first char of a special .Mark("in_qmark") // now we are really in a potential SPECIAL diff --git a/ParserTests/samples/TemplateTests.cs b/ParserTests/samples/TemplateTests.cs new file mode 100644 index 00000000..ca7d981c --- /dev/null +++ b/ParserTests/samples/TemplateTests.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using SimpleTemplate; +using sly.parser; +using sly.parser.generator; +using Xunit; + +namespace ParserTests.samples +{ + public class TemplateTests + { + + public Parser GetParser() + { + var builder = new ParserBuilder(); + var instance = new TemplateParser(); + var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "template"); + Assert.False(build.IsError); + Assert.NotNull(build.Result); + return build.Result; + } + + [Fact] + public void BasicTemplateTest() + { + var parser = GetParser(); + var source = @"hello-{=world=}-billy-{% if (a == 1.0) %}-bob-{%else%}-boubou-{%endif%}this is the end"; + + var context = new Dictionary() + { + { "world", "monde" }, + { "a", "1.0" } + }; + + var result = parser.ParseWithContext(source,context); + Assert.False(result.IsError); + Assert.Equal("hello-monde-billy--bob-this is the end",result.Result); + } + } +} \ No newline at end of file diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj index 67f7d8ea..fd0e2680 100644 --- a/samples/ParserExample/ParserExample.csproj +++ b/samples/ParserExample/ParserExample.csproj @@ -32,6 +32,7 @@ + diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index 595246dd..af3c7ae5 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -20,6 +20,7 @@ using ParserTests.Issue239; using ParserTests.lexer; using simpleExpressionParser; +using SimpleTemplate; using sly.lexer; using sly.lexer.fsm; using sly.parser; @@ -995,10 +996,57 @@ public static void TestIndentedParserNeverEnding() Assert.NotNull(ifthenelse.Else); Assert.Equal(2,ifthenelse.Else.Statements.Count); } + + private static void TestTemplate() + { + var lexerResult = LexerBuilder.BuildLexer(); + Assert.False(lexerResult.IsError); + var genericLexer = lexerResult.Result as GenericLexer; + var fsm = genericLexer.FSMBuilder.Fsm; + var fsmGraph = fsm.ToGraphViz(); + Console.WriteLine(fsmGraph); + + + + var source = @"hello - {= world =} - billy - {% if (a == 1.0) %} - bob - {%else%} - boubou - {%endif%}"; + + var tokens = genericLexer.Tokenize(source); + foreach (var token in tokens.Tokens.Tokens) + { + Console.WriteLine(token); + } + + var builder = new ParserBuilder(); + var instance = new TemplateParser(); + var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "template"); + if (build.IsOk) + { + var context = new Dictionary() + { + { "world", "monde" }, + { "a", "1.0" } + }; + var r = build.Result.ParseWithContext(source, context); + if (r.IsOk) + { + Console.WriteLine(r.Result); + } + else + { + r.Errors.ForEach(x => Console.WriteLine(x.ErrorMessage)); + } + } + else + { + build.Errors.ForEach(x => Console.WriteLine(x.Message)); + } + + } private static void Main(string[] args) { - testErrors(); + TestTemplate(); + // testErrors(); //TestContextualParser(); //TestTokenCallBacks(); //test104(); @@ -1038,7 +1086,7 @@ private static void Main(string[] args) // TestChannels(); //TestIndentedParserNeverEnding(); //TestLexerModes(); - TestXmlParser(); + // TestXmlParser(); } private static void testManySugar() diff --git a/samples/SimpleTemplate/SimpleTemplate.csproj b/samples/SimpleTemplate/SimpleTemplate.csproj new file mode 100644 index 00000000..2fb42ea7 --- /dev/null +++ b/samples/SimpleTemplate/SimpleTemplate.csproj @@ -0,0 +1,11 @@ + + + + netstandard2.0 + + + + + + + diff --git a/samples/SimpleTemplate/TemplateLexer.cs b/samples/SimpleTemplate/TemplateLexer.cs new file mode 100644 index 00000000..b37da699 --- /dev/null +++ b/samples/SimpleTemplate/TemplateLexer.cs @@ -0,0 +1,161 @@ +using System; +using sly.lexer; + +namespace SimpleTemplate +{ + public enum TemplateLexer + { + + NOT_A_TOKEN = 0, + + #region TEXT + + [AllExcept("{%", "{=")] + // [AllExcept("{%")] + TEXT, + + [Sugar("{%")] [Push("code")] OPEN_CODE, + + [Sugar("{=")] [Push("value")] OPEN_VALUE, + + + + + + #endregion + + #region value + + [AlphaId] + [Mode("value")] + [Mode("code")] + ID, + + [Sugar("=}")] + [Mode("value")] + [Pop] + CLOSE_VALUE, + + #endregion + + #region code + + [Sugar("%}")] + [Mode("code")] + [Pop] + CLOSE_CODE, + + [Keyword("if")] + [Mode("code")] + IF, + + [Keyword("endif")] + [Mode("code")] + ENDIF, + + [Keyword("else")] + [Mode("code")] + ELSE, + + #region literals + + [String()] + [Mode("code")] + STRING, + + // [Int()] + // [Mode("code")] + // INT, + + [Double()] + [Mode("code")] + DOUBLE, + + [Lexeme(GenericToken.KeyWord, "TRUE")] + [Lexeme(GenericToken.KeyWord, "true")] + [Mode("code")] + TRUE, + + [Lexeme(GenericToken.KeyWord, "FALSE")] + [Lexeme(GenericToken.KeyWord, "false")] + [Mode("code")] + FALSE, + + + + #endregion + + #region operators 30 -> 49 + + [Sugar( ">")] + [Mode("code")] + GREATER = 30, + + [Sugar( "<")] + [Mode("code")] + LESSER = 31, + + [Sugar( "==")] + [Mode("code")] + EQUALS = 32, + + [Sugar( "!=")] + [Mode("code")] + DIFFERENT = 33, + + [Sugar( ".")] + [Mode("code")] + CONCAT = 34, + + [Sugar( ":=")] + [Mode("code")] + ASSIGN = 35, + + [Sugar( "+")] + [Mode("code")] + PLUS = 36, + + [Sugar( "-")] + [Mode("code")] + MINUS = 37, + + + [Sugar( "*")] + [Mode("code")] + TIMES = 38, + + [Sugar( "/")] + [Mode("code")] + DIVIDE = 39, + + + #endregion + + #region sugar 100 -> 150 + + [Sugar("(")] + [Mode("code")] + OPEN_PAREN, + + [Sugar(")")] + [Mode("code")] + CLOSE_PAREN, + + [Lexeme(GenericToken.KeyWord, "NOT")] [Lexeme(GenericToken.KeyWord, "not")] + [Mode("code")] + NOT, + + [Lexeme(GenericToken.KeyWord, "AND")] [Lexeme(GenericToken.KeyWord, "and")] + [Mode("code")] + AND, + + [Lexeme(GenericToken.KeyWord, "OR")] [Lexeme(GenericToken.KeyWord, "or")] + [Mode("code")] + OR, + + #endregion + + #endregion + + } +} \ No newline at end of file diff --git a/samples/SimpleTemplate/TemplateParser.cs b/samples/SimpleTemplate/TemplateParser.cs new file mode 100644 index 00000000..af2dd706 --- /dev/null +++ b/samples/SimpleTemplate/TemplateParser.cs @@ -0,0 +1,260 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace SimpleTemplate +{ + public class TemplateParser + { + + [Production("template: item*")] + public string Template(List items, Dictionary context) + { + return string.Join("",items); + } + + [Production("item : TEXT")] + public string Text(Token text, Dictionary context) + { + return text.Value; + } + + [Production("item :OPEN_VALUE[d] ID CLOSE_VALUE[d]")] + public string Value(Token value, Dictionary context) + { + + if (context.TryGetValue(value.Value, out var val)) + { + return val; + } + + return ""; + } + + [Production("item : if item (else item)? endif ")] + public string Conditional(string cond, string thenBlock, ValueOption> elseBlock, + string endif, Dictionary context) + { + if (cond == "1") + { + return thenBlock; + } + else + { + var gg = elseBlock.Match( + (Group group) => + { + return group; + }, + () => + { + var g = new Group(); + g.Add("item", ""); + return g; + }); + return gg.Value("item"); + } + } + + [Production("if : OPEN_CODE[d] IF[d] OPEN_PAREN[d] TemplateParser_expressions CLOSE_PAREN[d] CLOSE_CODE[d]")] + public string If(string condition, Dictionary context) + { + return condition; + } + + [Production("else :OPEN_CODE[d] ELSE[d] CLOSE_CODE[d]")] + public string Else(Dictionary context) + { + return ""; + } + + [Production("endif :OPEN_CODE[d] ENDIF[d] CLOSE_CODE[d]")] + public string EndIf(Dictionary context) + { + return ""; + } + + + + + + + + #region COMPARISON OPERATIONS + + [Infix("LESSER", Associativity.Right, 50)] + [Infix("GREATER", Associativity.Right, 50)] + [Infix("EQUALS", Associativity.Right, 50)] + [Infix("DIFFERENT", Associativity.Right, 50)] + public string binaryComparisonExpression(string left, Token operatorToken, + string right, Dictionary context) + { + int comparison = left.CompareTo(right); + + switch (operatorToken.TokenID) + { + case TemplateLexer.LESSER: + { + return comparison < 0 ? "1" : "0"; + break; + } + case TemplateLexer.GREATER: + { + return comparison > 0 ? "1" : "0"; + } + case TemplateLexer.EQUALS: + { + return comparison == 0 ? "1" : "0"; + } + case TemplateLexer.DIFFERENT: + { + return comparison != 0 ? "1" : "0"; + } + } + + return "0"; + } + + #endregion + + #region STRING OPERATIONS + + [Operation((int) TemplateLexer.CONCAT, Affix.InFix, Associativity.Right, 10)] + public string binaryStringExpression(string left, Token operatorToken, string right, Dictionary context) + { + return left + right; + } + + #endregion + + #region OPERANDS + + + [Production("primary: DOUBLE")] + public string PrimaryInt(Token intToken, Dictionary context) + { + return intToken.Value; + } + + + [Production("primary: TRUE")] + [Production("primary: FALSE")] + public string PrimaryBool(Token boolToken, Dictionary context) + { + return bool.Parse(boolToken.StringWithoutQuotes) ? "1" : "0"; + } + + + [Production("primary: STRING")] + public string PrimaryString(Token stringToken, Dictionary context) + { + return stringToken.StringWithoutQuotes; + } + + + [Production("primary: ID")] + public string PrimaryId(Token varToken, Dictionary context) + { + string result = ""; + context.TryGetValue(varToken.Value, out result); + return result; + } + + [Operand] + [Production("operand: primary")] + public string Operand(string prim, Dictionary context) + { + return prim; + } + + #endregion + + #region NUMERIC OPERATIONS + + [Operation((int) TemplateLexer.PLUS, Affix.InFix, Associativity.Right, 10)] + [Operation((int) TemplateLexer.MINUS, Affix.InFix, Associativity.Right, 10)] + public string binaryTermNumericExpression(string left, Token operatorToken, + string right, Dictionary context) + { + double l = double.Parse(left); + double r = double.Parse(right); + + switch (operatorToken.TokenID) + { + case TemplateLexer.PLUS: + { + return (l + r).ToString(); + } + case TemplateLexer.MINUS: + { + return (l - r).ToString(); + } + } + + + return ""; + } + + [Operation((int) TemplateLexer.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation((int) TemplateLexer.DIVIDE, Affix.InFix, Associativity.Right, 50)] + public string binaryFactorNumericExpression(string left, Token operatorToken, + string right, Dictionary context) + { + double l = double.Parse(left); + double r = double.Parse(right); + + switch (operatorToken.TokenID) + { + case TemplateLexer.TIMES: + { + return (l * r).ToString(); + } + case TemplateLexer.DIVIDE: + { + return (l / r).ToString(); + } + } + + return ""; + } + + [Prefix((int) TemplateLexer.MINUS, Associativity.Right, 100)] + public string unaryNumericExpression(Token operation, string value, Dictionary context) + { + if (double.TryParse(value, out double x)) + { + return (-x).ToString(); + } + + return "0"; + } + + #endregion + + + #region BOOLEAN OPERATIONS + + [Operation((int) TemplateLexer.OR, Affix.InFix, Associativity.Right, 10)] + public string binaryOrExpression(string left, Token operatorToken, string right, Dictionary context) + { + return left == "1" || right == "1" ? "1" : "0"; + } + + [Operation((int) TemplateLexer.AND, Affix.InFix, Associativity.Right, 50)] + public string binaryAndExpression(string left, Token operatorToken, string right, Dictionary context) + { + return left == "1" && right == "1" ? "1" : "0"; + } + + [Operation((int) TemplateLexer.NOT, Affix.PreFix, Associativity.Right, 100)] + public string binaryOrExpression(Token operatorToken, string value, Dictionary context) + { + return value == "1" ? "0" : "1"; + } + + #endregion + } +} \ No newline at end of file diff --git a/samples/XML/MinimalXmlLexer.cs b/samples/XML/MinimalXmlLexer.cs index a712114f..0d91fa80 100644 --- a/samples/XML/MinimalXmlLexer.cs +++ b/samples/XML/MinimalXmlLexer.cs @@ -4,7 +4,7 @@ namespace XML { [Lexer()] - [Modes("one","two")] + [Modes("tag","pi")] public enum MinimalXmlLexer { NOT_A_TOKEN = 0, diff --git a/sly.sln b/sly.sln index 060febf3..6552b747 100644 --- a/sly.sln +++ b/sly.sln @@ -37,6 +37,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "postProcessedLexerParser", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XML", "samples\XML\XML.csproj", "{81E72CFA-A6D6-4DB4-B3B5-E167064E7858}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleTemplate", "samples\SimpleTemplate\SimpleTemplate.csproj", "{BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}" +EndProject Global GlobalSection(Performance) = preSolution HasPerformanceSessions = true @@ -114,6 +116,10 @@ Global {81E72CFA-A6D6-4DB4-B3B5-E167064E7858}.Debug|Any CPU.Build.0 = Debug|Any CPU {81E72CFA-A6D6-4DB4-B3B5-E167064E7858}.Release|Any CPU.ActiveCfg = Release|Any CPU {81E72CFA-A6D6-4DB4-B3B5-E167064E7858}.Release|Any CPU.Build.0 = Release|Any CPU + {BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -129,6 +135,7 @@ Global {B1FAADF0-208D-4A0A-A24D-D7BF4E33DCB4} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33} {8F703C3C-0D10-4F43-AC2E-14843D59E460} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33} {81E72CFA-A6D6-4DB4-B3B5-E167064E7858} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33} + {BC5020CA-6BFC-4895-9A1E-99B4A0AEDB11} = {36ED7F1A-2E81-4A71-81FE-A357E5840A33} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {43254130-CF3E-480E-952F-E50CA5D2E417} diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs index db4b5bf0..8f463d6d 100644 --- a/sly/lexer/GenericLexer.cs +++ b/sly/lexer/GenericLexer.cs @@ -5,6 +5,7 @@ using sly.buildresult; using sly.i18n; using sly.lexer.fsm; +using sly.lexer.fsm.transitioncheck; namespace sly.lexer { @@ -832,8 +833,16 @@ public void AddSugarLexem(IN token, BuildResult> buildResult, string return match; }; + Func precond = (int i) => + { + return (ReadOnlyMemory value) => + { + return value.Length == i+1; + }; + }; + FSMBuilder.GoTo(start); - for (var i = 0; i < specialValue.Length; i++) FSMBuilder.SafeTransition(specialValue[i]); + for (var i = 0; i < specialValue.Length; i++) FSMBuilder.SafeTransition(specialValue[i],precond(i)); FSMBuilder.End(GenericToken.SugarToken, isLineEnding) .CallBack(callback); } @@ -850,11 +859,73 @@ public void AddAllExcept(IN token, BuildResult> buildResult, string[] FSMBuilder.GoTo(start); - FSMBuilder.ExceptTransition(exceptions.First().ToCharArray()) - .Mark(in_all_except+"_"+allExceptCounter) + var allExceptChars0 = exceptions.Select(x => x.First()).Distinct().ToArray(); + + Func GetEndLabel = (int exception, int exceptionIndex) => + { + if (exception < 0 || exceptionIndex < 0) + { + return $"{in_all_except}_text_{allExceptCounter}"; + } + + return $"{in_all_except}_{exception}_{exceptionIndex}_{allExceptCounter}"; + }; + + var allExceptChars1 = exceptions.Where(x => x.Length >= 2).Select(x => x[1]).Distinct().ToArray(); + + FSMBuilder.ExceptTransition(allExceptChars0) + .Mark(GetEndLabel(-1,-1)) .End(GenericToken.AllExcept) .CallBack(callback); - FSMBuilder.ExceptTransitionTo(exceptions.First().ToCharArray(),in_all_except+"_"+allExceptCounter); + FSMBuilder.ExceptTransitionTo(allExceptChars0,GetEndLabel(-1,-1)); + for (int i = 0; i < exceptions.Length; i++) + { + string exception = exceptions[i]; + for (int j = 0; j < exception.Length-1; j ++) + { + char exceptionChar = exception[j]; + var end = GetEndLabel(i, j); + var startNode = GetEndLabel(i, j - 1); + if (j == 0) + { + FSMBuilder.GoTo(startNode); + FSMBuilder.SafeTransition(exceptionChar); + FSMBuilder.Mark(end); + // FSMBuilder.End(GenericToken.AllExcept) + // .CallBack(callback); + + startNode = start; + } + + + FSMBuilder.GoTo(startNode); + + if (j < exception.Length - 1) + { + FSMBuilder.TransitionToAndMark(new[] { exceptionChar }, end); + + + var transition = FSMBuilder.GetTransition(exception[j + 1]); + if (transition != null) + { + if (transition.Check is TransitionAnyExcept except) + { + except.AddException(exception[j+1]); + } + } + else + { + FSMBuilder.ExceptTransitionTo(new[] { exception[j + 1] }, GetEndLabel(-1, -1)); + } + // TODO : si on a un except qui match le lien retour vers text + // alors ajouter exception[j+1] dans la transition + } + + //FSMBuilder.Mark(GetEndLabel(i, j)); + + } + } + allExceptCounter++; } diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index 67b1a6cd..557f3032 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -234,7 +234,7 @@ private static Dictionary>> GetSub } var push = enumValue.GetAttributesOfType(); - if (push != null && push.Length == 1) + if (push != null && push.Length >= 1) { attribute.Value.ForEach(x => { @@ -242,22 +242,14 @@ private static Dictionary>> GetSub x.Pushtarget = push.First().TargetMode; }); } - else if (push.Length > 1) - { - // TODO : error can not push many mode - } var pop = enumValue.GetAttributesOfType(); - if (pop != null && pop.Length == 1) + if (pop != null && pop.Length >= 1) { attribute.Value.ForEach(x => { x.IsPop = true; }); } - else if (pop.Length > 1) - { - // TODO : error only pop once - } } } diff --git a/sly/lexer/attributes/AllExcept.cs b/sly/lexer/attributes/AllExcept.cs index b92dcdb0..23daa3e3 100644 --- a/sly/lexer/attributes/AllExcept.cs +++ b/sly/lexer/attributes/AllExcept.cs @@ -2,7 +2,7 @@ namespace sly.lexer { public class AllExceptAttribute : LexemeAttribute { - public AllExceptAttribute(string exceptions) : base(GenericToken.AllExcept, exceptions) + public AllExceptAttribute(params string[] exceptions) : base(GenericToken.AllExcept, exceptions) { } } diff --git a/sly/lexer/fsm/FSMLexerBuilder.cs b/sly/lexer/fsm/FSMLexerBuilder.cs index 29f1c5e8..5f5e846a 100644 --- a/sly/lexer/fsm/FSMLexerBuilder.cs +++ b/sly/lexer/fsm/FSMLexerBuilder.cs @@ -142,7 +142,7 @@ public FSMLexerBuilder Push(string toMode) return this; } - // TODO : later pop to a specific mode + // TODO : later pop to a specific mode ? public FSMLexerBuilder Pop(string toMode = null) { if (Fsm.HasState(CurrentState)) @@ -179,6 +179,12 @@ public FSMLexerBuilder SafeTransition(char input) return this; } + public FSMTransition GetTransition(char input) + { + var transition = Fsm.GetTransition(CurrentState, input); + return transition; + } + public FSMLexerBuilder SafeTransition(char input, TransitionPrecondition precondition) { var transition = Fsm.GetTransition(CurrentState, input); @@ -515,6 +521,22 @@ public FSMLexerBuilder TransitionTo(char[] inputs, string toNodeMark) var toNode = Marks[toNodeMark]; return TransitionTo(inputs, toNode); } + + public FSMLexerBuilder TransitionToAndMark(char[] inputs, string toNodeMark) + { + int toNode = -1; + if (Marks.TryGetValue(toNodeMark, out toNode)) + { + TransitionTo(inputs, toNode); + } + else + { + Transition(inputs); + Mark(toNodeMark); + } + + return this; + } public FSMLexerBuilder TransitionTo(char input, string toNodeMark, TransitionPrecondition precondition) @@ -547,7 +569,7 @@ public FSMLexerBuilder ExceptTransitionTo(char[] exceptions, string toNodeMar var toNode = Marks[toNodeMark]; return ExceptTransitionTo(exceptions, toNode); } - + public FSMLexerBuilder ExceptTransitionTo(char[] exceptions, string toNodeMark, TransitionPrecondition precondition) { diff --git a/sly/lexer/fsm/transitioncheck/TransitionAnyExcept.cs b/sly/lexer/fsm/transitioncheck/TransitionAnyExcept.cs index af69dbe6..4d1227ca 100644 --- a/sly/lexer/fsm/transitioncheck/TransitionAnyExcept.cs +++ b/sly/lexer/fsm/transitioncheck/TransitionAnyExcept.cs @@ -14,6 +14,11 @@ public TransitionAnyExcept(params char[] tokens) TokenExceptions.AddRange(tokens); } + public void AddException(char exception) + { + TokenExceptions.Add(exception); + } + public TransitionAnyExcept(TransitionPrecondition precondition, params char[] tokens) { TokenExceptions = new List();