hexa : initial

b3b00 · Jul 2, 2024 · 5750d7a · 5750d7a
1 parent 9ad4e80
commit 5750d7a
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 11 deletions.
diff --git a/src/sly/lexer/GenericLexer.cs b/src/sly/lexer/GenericLexer.cs
@@ -68,6 +68,7 @@ public Config()
         public const string in_int = "in_int";
         public const string start_double = "start_double";
         public const string in_double = "in_double";
+        public const string in_hexa = "in_hexa";
         public const string in_identifier = "in_identifier";
         public const string token_property = "token";
         public const string DerivedToken = "derivedToken";
@@ -541,6 +542,36 @@ public void AddDouble(IN token, string separator, BuildResult<ILexer<IN>> result
             FSMBuilder.Fsm.DecimalSeparator = separatorChar;
         }
 
+        public void AddHexa(IN token, string prefix, BuildResult<ILexer<IN>> result)
+        {
+            NodeCallback<GenericToken> callback = match =>
+            {
+                IN derivedToken = token;
+
+                match.HexaPrefix = prefix;
+
+                match.Properties[DerivedToken] = derivedToken;
+
+                return match;
+            };
+            FSMBuilder.GoTo(start);
+            if (!string.IsNullOrEmpty(prefix))
+            {
+                FSMBuilder.GoTo(start);
+
+                for (int i = 0; i < prefix.Length; i++)
+                {
+                    FSMBuilder.SafeTransition(prefix[i]);
+                }
+
+                FSMBuilder.MultiRangeTransition(('0', '9'), ('a', 'f'), ('A', 'F'))
+                    .Mark(in_hexa)
+                    .MultiRangeTransitionTo(in_hexa, ('0', '9'), ('a', 'f'), ('A', 'F'))
+                    .End(GenericToken.Hexa)
+                    .CallBack(callback);
+            }
+        }
+
         public void AddDate(IN token, DateFormat format, char separator, LexemeAttribute doubleLexeme,
             BuildResult<ILexer<IN>> result)
         {
@@ -1143,6 +1174,7 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
             tok.Position = inTok.Position;
             tok.Discarded = inTok.Discarded;
             tok.StringDelimiter = match.StringDelimiterChar;
+            tok.hexaPrefix = match.HexaPrefix;
             tok.TokenID = match.Properties.TryGetValue(DerivedToken, out var property) ? (IN)property : default;
             tok.IsLineEnding = match.IsLineEnding;
             tok.IsEOS = match.IsEOS;

diff --git a/src/sly/lexer/GenericToken.cs b/src/sly/lexer/GenericToken.cs
@@ -5,6 +5,7 @@ public enum GenericToken
         Default,
         Identifier,
         Int,
+        Hexa,
         Double,
         Date,
         KeyWord,

diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs
@@ -501,6 +501,17 @@ private static BuildResult<ILexer<IN>> BuildGenericLexer<IN>(IDictionary<IN, Lis
                             lexer.AddLexeme(lexeme.GenericToken, tokenID);
                         }
 
+                        if (lexeme.IsHexa)
+                        {
+                            string prefix = "0x";
+                            if (lexeme.GenericTokenParameters != null && lexeme.GenericTokenParameters.Any())
+                            {
+                               prefix = lexeme.GenericTokenParameters[0];
+                            }
+
+                            lexer.AddHexa(tokenID, prefix, result);
+                        }
+
                         if (lexeme.IsKeyWord)
                         {
                             foreach (var param in lexeme.GenericTokenParameters)

diff --git a/src/sly/lexer/Token.cs b/src/sly/lexer/Token.cs
@@ -29,6 +29,9 @@ public class Token<T> where T:struct
         [JsonIgnore]
         public bool Notignored;
 
+        [JsonIgnore] public string hexaPrefix = "0x";
+
+
 
         public Token(T token, string value, LexerPosition position, 
             CommentType commentType = CommentType.Single, int? channel = null, char decimalSeparator = '.' ) : this(token,new ReadOnlyMemory<char>(value.ToCharArray()),position,commentType,channel, decimalSeparator:decimalSeparator)
@@ -231,6 +234,9 @@ public string StringWithoutQuotes
 
         [JsonIgnore]
         public int IntValue => int.Parse(Value);
+
+        [JsonIgnore]
+        public long HexaIntValue => Convert.ToInt32(Value.Substring(hexaPrefix.Length), 16);
 
         [JsonIgnore]
         public DateTime DateTimeValue { get; set; }
@@ -257,11 +263,8 @@ public char CharValue  {
                 var result = Value;
                 if (CharDelimiter != (char) 0)
                 {
-                    if (result.StartsWith(CharDelimiter.ToString()))  {
-                        result = result.Substring(1);
-                    }
-                    if (result.EndsWith(CharDelimiter.ToString())) {
-                        result = result.Substring(0, result.Length - 1);
+                    if (result.StartsWith(CharDelimiter.ToString()) && result.EndsWith(CharDelimiter.ToString()) && result.Length > 2) {
+                        result = result.Substring(1, result.Length - 1);
                     }
                 }
                 return result[0];

diff --git a/src/sly/lexer/attributes/HexaAttribute.cs b/src/sly/lexer/attributes/HexaAttribute.cs
@@ -0,0 +1,11 @@
+namespace sly.lexer;
+
+public class HexaAttribute : LexemeAttribute
+{
+
+
+    public HexaAttribute(string hexaPrefix = "0x", int channel = Channels.Main) : base(GenericToken.Hexa,
+        channel:channel, parameters:hexaPrefix)
+    {
+    }
+}
diff --git a/src/sly/lexer/attributes/LexemeAttribute.cs b/src/sly/lexer/attributes/LexemeAttribute.cs
@@ -86,6 +86,8 @@ public LexemeAttribute(GenericToken generic, IdentifierType idType, string start
         public bool IsDouble => GenericToken == GenericToken.Double;
         public bool IsUpTo => GenericToken == GenericToken.UpTo;
 
+        public bool IsHexa => GenericToken == GenericToken.Hexa;
+
         public bool IsDate => GenericToken == GenericToken.Date;
     }
 }
diff --git a/src/sly/lexer/fsm/FSMMatch.cs b/src/sly/lexer/fsm/FSMMatch.cs
@@ -9,6 +9,8 @@ public class FSMMatch<N>  where N : struct
 
         public char StringDelimiterChar { get; set; }
 
+        public string HexaPrefix { get; set; }
+
         public char DecimalSeparator { get; set; }
         public bool IsSuccess { get; set; }
 

diff --git a/src/sly/parser/generator/visitor/MermaidEBNFSyntaxTreeVisitor.cs b/src/sly/parser/generator/visitor/MermaidEBNFSyntaxTreeVisitor.cs
@@ -119,14 +119,18 @@ public MermaidNode VisitLeaf(Token<IN> token)
             }
             else if (token.IsExplicit)
             {
-                return Leaf(token.Value);
+                return Leaf(token.StringWithoutQuotes);
             }
-            return Leaf(token.TokenID, token.Value);
+            return Leaf(token.TokenID, token.StringWithoutQuotes);
         }
 
         private MermaidNode Leaf(IN type, string value)
         {
-            string label = type.ToString();
+            if (value.Contains("pi"))
+            {
+                Console.WriteLine("3.14");
+            }
+            string label = "\""+type.ToString();
             if (label == "0")
             {
                 label = "";
@@ -135,7 +139,7 @@ private MermaidNode Leaf(IN type, string value)
             {
                 label += "\n";
             }
-            label += $"'{value}'";
+            label += $"'{value}'\"";
             var node = new MermaidNode(NodeCounter.ToString())
             {
                 // Set all available properties
@@ -153,7 +157,7 @@ private MermaidNode Leaf(IN type, string value)
         private MermaidNode Leaf(string value)
         {
             string label = "";
-            label += $"'{value}'";
+            label += $@"""'{value}'""";
             var node = new MermaidNode(NodeCounter.ToString())
             {
                 // Set all available properties
@@ -171,7 +175,7 @@ private MermaidNode Leaf(string value)
 
         private string GetNodeLabel(SyntaxNode<IN> node)
         {
-            string label = node.Name;
+            string label = $@"""{node.Name}""";
             return label;
         }
 

diff --git a/tests/ParserTests/IssueTest/IssueLexer.cs b/tests/ParserTests/IssueTest/IssueLexer.cs
@@ -0,0 +1,6 @@
+namespace ParserTests.IssueTest;
+
+public enum IssueLexer
+{
+
+}
diff --git a/tests/ParserTests/IssueTest/IssueTest.cs b/tests/ParserTests/IssueTest/IssueTest.cs
@@ -0,0 +1,6 @@
+namespace ParserTests.IssueTest;
+
+public class IssueTest
+{
+
+}
diff --git a/tests/ParserTests/lexer/GenericLexerTests.cs b/tests/ParserTests/lexer/GenericLexerTests.cs
@@ -17,6 +17,19 @@
 namespace ParserTests.lexer
 {
 
+    public enum HexaTokenDefault
+    {
+        [Hexa]HEXA,
+    }
+
+    public enum HexaTokenConflictId
+    {
+        [Hexa("hexa")]HEXA,
+
+        [AlphaId] ID
+    }
+
+
     public enum DateTokenEnglishDashed
     {
         [Date(DateFormat.YYYYMMDD, '-')] DATE,
@@ -1175,6 +1188,46 @@ public void TestDateAndExpressions()
 
         }
 
+        [Fact]
+        public void TestDefaultHexa()
+        {
+            var lexerRes = LexerBuilder.BuildLexer(new BuildResult<ILexer<HexaTokenDefault>>());
+            Check.That(lexerRes.IsError).IsFalse();
+            var lexer = lexerRes.Result;
+
+            var lexerResult = lexer.Tokenize("0xAA 0x123 0xAb89CF");
+            Check.That(lexerResult).IsOkLexing();
+            Check.That(lexerResult.Tokens).IsNotNull();
+            Check.That(lexerResult.Tokens).CountIs(4);
+            Check.That(lexerResult.Tokens.Extracting(x => (x.TokenID,x.HexaIntValue)).Take(3)).IsEqualTo(new List<(HexaTokenDefault,long)>()
+            {
+                (HexaTokenDefault.HEXA,Convert.ToInt32("AA",16)),
+                (HexaTokenDefault.HEXA,Convert.ToInt32("123",16)),
+                (HexaTokenDefault.HEXA,Convert.ToInt32("AB89CF",16))
+            });
+        }
+
+        [Fact]
+        public void TestHexaConflictId()
+        {
+            var lexerRes = LexerBuilder.BuildLexer(new BuildResult<ILexer<HexaTokenConflictId>>());
+            Check.That(lexerRes.IsError).IsFalse();
+            var lexer = lexerRes.Result;
+
+            var lexerResult = lexer.Tokenize("hexaAA hexa123 hexaAb89CF hello");
+            Check.That(lexerResult).IsOkLexing();
+            Check.That(lexerResult.Tokens).IsNotNull();
+            Check.That(lexerResult.Tokens).CountIs(5);
+            Check.That(lexerResult.Tokens.Extracting(x => (x.TokenID,x.HexaIntValue)).Take(3)).IsEqualTo(new List<(HexaTokenConflictId,long)>()
+            {
+                (HexaTokenConflictId.HEXA,Convert.ToInt32("AA",16)),
+                (HexaTokenConflictId.HEXA,Convert.ToInt32("123",16)),
+                (HexaTokenConflictId.HEXA,Convert.ToInt32("AB89CF",16))
+            });
+            Check.That(lexerResult.Tokens[3].TokenID).IsEqualTo(HexaTokenConflictId.ID);
+            Check.That(lexerResult.Tokens[3].Value).IsEqualTo("hello");
+        }
+
         [Fact]
         public void TestI18nLexerError() // issue #380
         {
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ public enum GenericToken @@
             Default,
             Identifier,
             Int,
+            Hexa,
             Double,
             Date,
             KeyWord,
@@ Expand Down @@