fix fstrings : bug with upto tokens and leading whitspaces

b3b00 · Nov 5, 2024 · b8d255e · b8d255e
1 parent 0514b01
commit b8d255e
Show file tree

Hide file tree

Showing 15 changed files with 103 additions and 174 deletions.
diff --git a/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs b/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs
@@ -143,24 +143,49 @@ public WhileAST skipStmt(WhileAST expression)
 
         #region OPERANDS
 
+        public BinaryOperation BuildConcat(List<Expression> operands)
+        {
+            if (operands.Count == 1)
+            {
+                var operation = new BinaryOperation(operands[0], BinaryOperator.CONCAT, new StringConstant("",null));
+                return operation;
+            }
+            else if (operands.Count == 2)
+            {
+                var operation = new BinaryOperation(operands[0], BinaryOperator.CONCAT, operands[1]);
+                return operation;
+            }
+            else if (operands.Count > 2)
+            {
+                var right = BuildConcat(operands.Skip(1).ToList());
+                var operation = new BinaryOperation(operands[0], BinaryOperator.CONCAT, right);
+                return operation;
+            }
+
+            return null;
+        }
+
         // fstrings 
         [Production("primary : OPEN_FSTRING[d] fstring_element* CLOSE_FSTRING[d]")]
         public WhileAST fstring(List<WhileAST> elements)
         {
-            var fstring = new FString(elements.Cast<FStringElement>().ToList(), elements.First().Position);
-            return fstring;
+            if (elements.Count == 1)
+            {
+                return elements[0];
+            }
+            return BuildConcat(elements.Cast<Expression>().ToList());
         }
 
         [Production("fstring_element : FSTRING_CONTENT")]
         public WhileAST FStringContent(Token<IndentedWhileTokenGeneric> element)
         {
-            return new FStringElement(new StringConstant(element.Value),element.Position);
+            return new StringConstant(element.Value,element.Position);
         }
 
-        [Production("fstring_element : OPEN_FSTRING_EXPPRESSION[d] IDENTIFIER CLOSE_FSTRING_EXPPRESSION[d]")]
-        public WhileAST FStringExpression(Token<IndentedWhileTokenGeneric> element)
+        [Production("fstring_element : OPEN_FSTRING_EXPPRESSION[d] IndentedWhileParserGeneric_expressions CLOSE_FSTRING_EXPPRESSION[d]")]
+        public WhileAST FStringExpression(WhileAST expression)
         {
-            return new FStringElement(new Variable(element.Value),element.Position);
+            return expression;
         }
 
 

diff --git a/src/samples/IndentedWhile/parser/IndentedWhileTokenGeneric.cs b/src/samples/IndentedWhile/parser/IndentedWhileTokenGeneric.cs
@@ -60,28 +60,36 @@ public enum IndentedWhileTokenGeneric
 
         #region operators 30 -> 49
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, ">")] GREATER = 30,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "<")] LESSER = 31,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "==")]
         EQUALS = 32,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "!=")]
         DIFFERENT = 33,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, ".")] CONCAT = 34,
 
-        [Lexeme(GenericToken.SugarToken, ":=")]
-        ASSIGN = 35,
+
+        [Lexeme(GenericToken.SugarToken, ":=")] ASSIGN = 35,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "+")] PLUS = 36,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "-")] MINUS = 37,
 
-
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "*")] TIMES = 38,
 
+        [Mode(ModeAttribute.DefaultLexerMode, "fstringExpression")]
         [Lexeme(GenericToken.SugarToken, "/")] DIVIDE = 39,
 
         #endregion

diff --git a/src/samples/ParserExample/Program.cs b/src/samples/ParserExample/Program.cs
@@ -24,6 +24,7 @@
 using ParserTests.Issue414;
 using ParserTests.Issue495;
 using ParserTests.lexer;
+using ParserTests.samples;
 using simpleExpressionParser;
 using SimpleTemplate;
 using SimpleTemplate.model;
@@ -1352,7 +1353,8 @@ print a
         }
         private static void Main(string[] args)
         {
-            TestIssue495();
+            TestFStrings();
+            //TestIssue495();
             //testGenericLexerJson();
             // TestIssue487();
             //BenchSimpleExpression();
@@ -1834,6 +1836,12 @@ private static void TestIssue495()
               Check.That(parsed.Result).IsEqualTo("test=3 3");
 
           }
+
+          private static void TestFStrings()
+          {
+              IndentedWhileTests tests = new IndentedWhileTests();
+              tests.TestFString();
+          }
     }
 
     public enum TestGrammarToken

diff --git a/src/samples/SlowEOS/Program.cs b/src/samples/SlowEOS/Program.cs
@@ -10,7 +10,7 @@ public class Program
 
     private static Parser<SlowOnBadParseEosToken, object> GetParser(object instance)
     {
-        ParserBuilder<SlowOnBadParseEosToken, object> builder = new ParserBuilder<SlowOnBadParseEosToken, object>();
+        ParserBuilder<SlowOnBadParseEosToken, object> builder = new ParserBuilder<SlowOnBadParseEosToken, object>("en");
 
         var buildBaseLine = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
         if (buildBaseLine.IsOk)

diff --git a/src/samples/while/compiler/ExpressionTyper.cs b/src/samples/while/compiler/ExpressionTyper.cs
@@ -28,12 +28,6 @@ public WhileType TypeExpression(Expression expr, CompilerContext context)
                 variable.CompilerScope = context.CurrentScope;
                 return varType;
             }
-
-            if (expr is FString fstring)
-            {
-                return WhileType.STRING;
-            }
-
             throw new SignatureException($"unknow expression type ({expr.GetType().Name})");
         }
 

diff --git a/src/samples/while/model/FString.cs b/src/samples/while/model/FString.cs
diff --git a/src/samples/while/model/FStringElement.cs b/src/samples/while/model/FStringElement.cs
diff --git a/src/samples/while/model/StringConstant.cs b/src/samples/while/model/StringConstant.cs
@@ -7,9 +7,10 @@ namespace csly.whileLang.model
 {
     public class StringConstant : Expression
     {
-        public StringConstant(string value)
+        public StringConstant(string value, LexerPosition position)
         {
             Value = value;
+            Position = position;
         }
 
         public string Value { get; set; }

diff --git a/src/samples/while/parser/WhileParser.cs b/src/samples/while/parser/WhileParser.cs
@@ -149,7 +149,7 @@ public WhileAST PrimaryBool(Token<WhileToken> boolToken)
         [Production("primary: STRING")]
         public WhileAST PrimaryString(Token<WhileToken> stringToken)
         {
-            return new StringConstant(stringToken.StringWithoutQuotes);
+            return new StringConstant(stringToken.StringWithoutQuotes, stringToken.Position);
         }
 
         [Production("primary: IDENTIFIER")]

diff --git a/src/samples/while/parser/WhileParserGeneric.cs b/src/samples/while/parser/WhileParserGeneric.cs
@@ -148,7 +148,7 @@ public WhileAST PrimaryBool(Token<WhileTokenGeneric> boolToken)
         [Production("primary: STRING")]
         public WhileAST PrimaryString(Token<WhileTokenGeneric> stringToken)
         {
-            return new StringConstant(stringToken.StringWithoutQuotes);
+            return new StringConstant(stringToken.StringWithoutQuotes,stringToken.Position);
         }
 
         [Production("primary: IDENTIFIER")]

diff --git a/src/sly/lexer/GenericLexer.cs b/src/sly/lexer/GenericLexer.cs
@@ -221,8 +221,6 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> source)
                     tokens.Add(transcoded);
                 }
 
-
-
                 r = LexerFsm.Run(source, position);
                 if (r.IsNoIndent)
                 {

diff --git a/src/sly/lexer/fsm/FSMLexer.cs b/src/sly/lexer/fsm/FSMLexer.cs
@@ -114,7 +114,6 @@ public FSMTransition GetTransition(int nodeId, char token)
             return transition;
         }
 
-
         public void AddTransition(FSMTransition transition)
         {
             var transitions = new List<FSMTransition>();
@@ -252,17 +251,29 @@ public FSMMatch<N> Run(ReadOnlyMemory<char> source, LexerPosition lexerPosition)
 
             if (result != null)
             {
+                if ((result.Result as Token<GenericToken>).TokenID == GenericToken.UpTo)
+                {
+                    if (ignoredTokens.Count > 0)
+                    {
+                        int ignoredLength = ignoredTokens.Select(x => x.SpanValue.Length).Sum();
+                        int start = result.Result.Position.Index - ignoredLength; 
+                        result.Result.Position.Index  -= ignoredLength;
+                        var value = source.Slice(start, ignoredLength+result.Result.SpanValue.Length);
+                        result.Result.SpanValue = value;
+                    }
+                }
                 // Backtrack
                 var length = result.Result.Value.Length;
                 lexerPosition.Index = result.Result.Position.Index + length;
                 lexerPosition.Column = result.Result.Position.Column + length;
+                result.IgnoredTokens = ignoredTokens;
 
                 if (HasCallback(result.NodeId))
                 {
                     result = Callbacks[result.NodeId](result);
                 }
 
-                result.IgnoredTokens = ignoredTokens;
+
                 return result;
             }
-Original file line number
+Diff line change
@@ Expand Up @@
                         tokens.Add(transcoded);
                     }
                     r = LexerFsm.Run(source, position);
                     if (r.IsNoIndent)
                     {
@@ Expand Down @@