simple templating sample using lexer modes

b3b00 · Jun 29, 2022 · e752cee · e752cee
1 parent a7ee5ad
commit e752cee
Show file tree

Hide file tree

Showing 15 changed files with 639 additions and 21 deletions.
diff --git a/ParserTests/ParserTests.csproj b/ParserTests/ParserTests.csproj
@@ -27,6 +27,7 @@
     <ProjectReference Include="..\samples\indented\indented.csproj" />
     <ProjectReference Include="..\samples\postProcessedLexerParser\postProcessedLexerParser.csproj" />
     <ProjectReference Include="..\samples\SimpleExpressionParser\SimpleExpressionParser.csproj" />
+    <ProjectReference Include="..\samples\SimpleTemplate\SimpleTemplate.csproj" />
     <ProjectReference Include="..\samples\while\while.csproj" />
     <ProjectReference Include="..\samples\XML\XML.csproj" />
     <ProjectReference Include="..\sly\sly.csproj" />

diff --git a/ParserTests/lexer/Issue210Extensions.cs b/ParserTests/lexer/Issue210Extensions.cs
@@ -39,7 +39,7 @@ FSMMatch<GenericToken> Callback(FSMMatch<GenericToken> match)
 
 
                 lexer.FSMBuilder.GoTo(GenericLexer<Issue210Token>.start)
-                    .Transition('?')
+                    .SafeTransition('?')
                     .Mark("qmark")
                     .ExceptTransition(new[] {'?'}) // moving to first char of a special
                     .Mark("in_qmark") // now we are really in a potential SPECIAL

diff --git a/ParserTests/samples/TemplateTests.cs b/ParserTests/samples/TemplateTests.cs
@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+using SimpleTemplate;
+using sly.parser;
+using sly.parser.generator;
+using Xunit;
+
+namespace ParserTests.samples
+{
+    public class TemplateTests
+    {
+
+        public Parser<TemplateLexer, string> GetParser()
+        {
+            var builder = new ParserBuilder<TemplateLexer, string>();
+            var instance = new TemplateParser();
+            var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "template");
+            Assert.False(build.IsError);
+            Assert.NotNull(build.Result);
+            return build.Result;
+        }
+
+        [Fact]
+        public void BasicTemplateTest()
+        {
+            var parser = GetParser();
+            var source = @"hello-{=world=}-billy-{% if (a == 1.0) %}-bob-{%else%}-boubou-{%endif%}this is the end";
+
+            var context = new Dictionary<string, string>()
+            {
+                { "world", "monde" },
+                { "a", "1.0" }
+            };
+
+            var result = parser.ParseWithContext(source,context);
+            Assert.False(result.IsError);
+            Assert.Equal("hello-monde-billy--bob-this is the end",result.Result);
+        }
+    }
+}
diff --git a/samples/ParserExample/ParserExample.csproj b/samples/ParserExample/ParserExample.csproj
@@ -32,6 +32,7 @@
     <ProjectReference Include="..\..\sly\sly.csproj" />
     <ProjectReference Include="..\postProcessedLexerParser\postProcessedLexerParser.csproj" />
     <ProjectReference Include="..\SimpleExpressionParser\SimpleExpressionParser.csproj" />
+    <ProjectReference Include="..\SimpleTemplate\SimpleTemplate.csproj" />
     <ProjectReference Include="..\while\while.csproj" />
     <ProjectReference Include="..\XML\XML.csproj" />
   </ItemGroup>

diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs
@@ -20,6 +20,7 @@
 using ParserTests.Issue239;
 using ParserTests.lexer;
 using simpleExpressionParser;
+using SimpleTemplate;
 using sly.lexer;
 using sly.lexer.fsm;
 using sly.parser;
@@ -995,10 +996,57 @@ public static void TestIndentedParserNeverEnding()
             Assert.NotNull(ifthenelse.Else);
             Assert.Equal(2,ifthenelse.Else.Statements.Count);
         }
+
+        private static void TestTemplate()
+        {
+            var lexerResult = LexerBuilder.BuildLexer<TemplateLexer>();
+            Assert.False(lexerResult.IsError);
+            var genericLexer = lexerResult.Result as GenericLexer<TemplateLexer>;
+            var fsm = genericLexer.FSMBuilder.Fsm;
+            var fsmGraph = fsm.ToGraphViz();
+            Console.WriteLine(fsmGraph);
+
+
+
+            var source = @"hello - {= world =} - billy - {% if (a == 1.0) %} - bob - {%else%} - boubou - {%endif%}";
+
+            var tokens = genericLexer.Tokenize(source);
+            foreach (var token in tokens.Tokens.Tokens)
+            {
+                Console.WriteLine(token);
+            }
+
+            var builder = new ParserBuilder<TemplateLexer, string>();
+            var instance = new TemplateParser();
+            var build = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "template");
+            if (build.IsOk)
+            {
+                var context = new Dictionary<string, string>()
+                {
+                    { "world", "monde" },
+                    { "a", "1.0" }
+                };
+                var r = build.Result.ParseWithContext(source, context);
+                if (r.IsOk)
+                {
+                    Console.WriteLine(r.Result);
+                }
+                else
+                {
+                    r.Errors.ForEach(x => Console.WriteLine(x.ErrorMessage));
+                }
+            }
+            else
+            {
+                build.Errors.ForEach(x => Console.WriteLine(x.Message));
+            }
+
+        }
 
         private static void Main(string[] args)
         {
-            testErrors();
+            TestTemplate();
+            // testErrors();
             //TestContextualParser();
             //TestTokenCallBacks();
             //test104();
@@ -1038,7 +1086,7 @@ private static void Main(string[] args)
             // TestChannels();
             //TestIndentedParserNeverEnding();
             //TestLexerModes();
-            TestXmlParser();
+            // TestXmlParser();
         }
 
         private static void testManySugar()

diff --git a/samples/SimpleTemplate/SimpleTemplate.csproj b/samples/SimpleTemplate/SimpleTemplate.csproj
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netstandard2.0</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\..\sly\sly.csproj" />
+    </ItemGroup>
+
+</Project>
diff --git a/samples/SimpleTemplate/TemplateLexer.cs b/samples/SimpleTemplate/TemplateLexer.cs
@@ -0,0 +1,161 @@
+using System;
+using sly.lexer;
+
+namespace SimpleTemplate
+{
+    public enum TemplateLexer
+    {
+
+        NOT_A_TOKEN = 0,
+
+        #region TEXT
+
+        [AllExcept("{%", "{=")]
+        // [AllExcept("{%")]
+        TEXT,
+
+        [Sugar("{%")] [Push("code")] OPEN_CODE,
+
+        [Sugar("{=")] [Push("value")] OPEN_VALUE,
+
+
+
+
+
+    #endregion
+
+    #region value
+
+    [AlphaId]
+    [Mode("value")]
+    [Mode("code")]
+    ID,
+
+    [Sugar("=}")]
+    [Mode("value")]
+    [Pop]
+    CLOSE_VALUE,
+
+    #endregion
+
+    #region code
+
+    [Sugar("%}")]
+    [Mode("code")]
+    [Pop]
+    CLOSE_CODE,
+
+    [Keyword("if")]
+    [Mode("code")]
+    IF,
+
+    [Keyword("endif")]
+    [Mode("code")]
+    ENDIF,
+
+    [Keyword("else")]
+    [Mode("code")]
+    ELSE,
+
+    #region literals
+
+    [String()]
+    [Mode("code")]
+    STRING,
+
+    // [Int()]
+    // [Mode("code")]
+    // INT,
+
+    [Double()]
+    [Mode("code")]
+    DOUBLE,
+
+    [Lexeme(GenericToken.KeyWord, "TRUE")]
+    [Lexeme(GenericToken.KeyWord, "true")]
+    [Mode("code")]
+    TRUE,
+
+    [Lexeme(GenericToken.KeyWord, "FALSE")]
+    [Lexeme(GenericToken.KeyWord, "false")]
+    [Mode("code")]
+    FALSE,
+
+
+
+    #endregion
+
+    #region operators 30 -> 49
+
+    [Sugar( ">")]
+    [Mode("code")]
+    GREATER = 30,
+
+    [Sugar( "<")]
+    [Mode("code")]
+    LESSER = 31,
+
+    [Sugar( "==")]
+    [Mode("code")]
+    EQUALS = 32,
+
+    [Sugar( "!=")]
+    [Mode("code")]
+    DIFFERENT = 33,
+
+    [Sugar( ".")]
+    [Mode("code")]
+    CONCAT = 34,
+
+    [Sugar( ":=")]
+    [Mode("code")]
+    ASSIGN = 35,
+
+    [Sugar( "+")]
+    [Mode("code")]
+    PLUS = 36,
+
+    [Sugar( "-")]
+    [Mode("code")]
+    MINUS = 37,
+
+
+    [Sugar( "*")]
+    [Mode("code")]
+    TIMES = 38,
+
+    [Sugar( "/")]
+    [Mode("code")]
+    DIVIDE = 39,
+
+
+    #endregion
+
+    #region sugar 100 -> 150
+
+    [Sugar("(")]
+    [Mode("code")]
+    OPEN_PAREN,
+
+    [Sugar(")")]
+    [Mode("code")]
+    CLOSE_PAREN,
+
+    [Lexeme(GenericToken.KeyWord, "NOT")] [Lexeme(GenericToken.KeyWord, "not")]
+    [Mode("code")]
+    NOT,
+
+    [Lexeme(GenericToken.KeyWord, "AND")] [Lexeme(GenericToken.KeyWord, "and")]
+    [Mode("code")]
+    AND,
+
+    [Lexeme(GenericToken.KeyWord, "OR")] [Lexeme(GenericToken.KeyWord, "or")]
+    [Mode("code")]
+    OR,
+
+    #endregion
+
+    #endregion
+
+    }
+}