GD-213: Fix GdScript parser to handle string blocks (#215)

# Why
The GdScriptParser had problems with strings containing brackets and got
stuck.

# What
- fix parser to not tokenize strings and string blocks for brackets 
- Fixed function signature parsing so that spaces, tabs, and line breaks
are not lost before merging.

addons/gdUnit4/plugin.cfg

@@ -3,5 +3,5 @@
 name="gdUnit4"
 description="Unit Testing Framework for Godot Scripts"
 author="Mike Schulze"
-version="4.1.1"
+version="4.1.2"
 script="plugin.gd"

addons/gdUnit4/src/core/parse/GdScriptParser.gd

@@ -5,8 +5,10 @@ extends RefCounted
 const ALLOWED_CHARACTERS := "0123456789_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\""
 
 var TOKEN_NOT_MATCH := Token.new("")
-var TOKEN_SPACE := Token.new(" ")
-var TOKEN_COMMENT := Token.new("#")
+var TOKEN_SPACE := SkippableToken.new(" ")
+var TOKEN_TABULATOR := SkippableToken.new("\t")
+var TOKEN_NEW_LINE := SkippableToken.new("\n")
+var TOKEN_COMMENT := SkippableToken.new("#")
 var TOKEN_CLASS_NAME := Token.new("class_name")
 var TOKEN_INNER_CLASS := Token.new("class")
 var TOKEN_EXTENDS := Token.new("extends")
@@ -25,7 +27,7 @@ var TOKEN_BRACKET_OPEN := Token.new("(")
 var TOKEN_BRACKET_CLOSE := Token.new(")")
 var TOKEN_ARRAY_OPEN := Token.new("[")
 var TOKEN_ARRAY_CLOSE := Token.new("]")
-var TOKEN_NEW_LINE := Token.new("\n")
+
 var OPERATOR_ADD := Operator.new("+")
 var OPERATOR_SUB := Operator.new("-")
 var OPERATOR_MUL := Operator.new("*")
@@ -34,6 +36,8 @@ var OPERATOR_REMAINDER := Operator.new("%")
 
 var TOKENS := [
 	TOKEN_SPACE,
+	TOKEN_TABULATOR,
+	TOKEN_NEW_LINE,
 	TOKEN_COMMENT,
 	TOKEN_BRACKET_OPEN,
 	TOKEN_BRACKET_CLOSE,
@@ -52,7 +56,6 @@ var TOKENS := [
 	TOKEN_FUNCTION,
 	TOKEN_ARGUMENT_SEPARATOR,
 	TOKEN_FUNCTION_RETURN_TYPE,
-	TOKEN_NEW_LINE,
 	OPERATOR_ADD,
 	OPERATOR_SUB,
 	OPERATOR_MUL,
@@ -107,6 +110,9 @@ class Token extends RefCounted:
 	func is_token(token_name :String) -> bool:
 		return _token == token_name
 
+	func is_skippable() -> bool:
+		return false
+
 	func _to_string():
 		return "Token{" + _token + "}"
 
@@ -119,6 +125,16 @@ class Operator extends Token:
 		return "OperatorToken{%s}" % [_token]
 
 
+# A skippable token, is just a placeholder like space or tabs
+class SkippableToken extends Token:
+
+	func _init(p_token: String):
+		super(p_token)
+
+	func is_skippable() -> bool:
+		return true
+
+
 # Token to parse Fuzzers
 class FuzzerToken extends Token:
 	var _name: String
@@ -363,14 +379,15 @@ func parse_func_return_type(row: String) -> int:
 		return TYPE_NIL
 	return token.type()
 
+
 func parse_return_token(input: String) -> Token:
 	var index := input.rfind(TOKEN_FUNCTION_RETURN_TYPE._token)
 	if index == -1:
 		return TOKEN_NOT_MATCH
 	index += TOKEN_FUNCTION_RETURN_TYPE._consumed
 	var token := next_token(input, index)
-	if token == TOKEN_SPACE:
-		index += TOKEN_SPACE._consumed
+	while !token.is_variable() and token != TOKEN_NOT_MATCH:
+		index += token._consumed
 		token = next_token(input, index)
 	return token
 
@@ -385,8 +402,21 @@ func parse_arguments(input: String) -> Array[GdFunctionArgument]:
 	var in_function := false
 	while current_index < len(input):
 		token = next_token(input, current_index)
+		# fallback to not end in a endless loop
+		if token == TOKEN_NOT_MATCH:
+			var error : = """
+				Parsing Error: Invalid token at pos %d found.
+				Please report this error!
+				source_code:
+				--------------------------------------------------------------
+				%s
+				--------------------------------------------------------------
+			""".dedent() % [current_index, input]
+			push_error(error)
+			current_index += 1
+			continue
 		current_index += token._consumed
-		if token == TOKEN_SPACE:
+		if token.is_skippable():
 			continue
 		if token == TOKEN_BRACKET_OPEN:
 			in_function = true
@@ -418,39 +448,39 @@ func parse_arguments(input: String) -> Array[GdFunctionArgument]:
 				token = next_token(input, current_index)
 				current_index += token._consumed
 				#match token:
-				if token == TOKEN_SPACE:
-						continue
+				if token.is_skippable():
+					continue
 				elif token == TOKEN_ARGUMENT_TYPE:
+					token = next_token(input, current_index)
+					if token == TOKEN_SPACE:
+						current_index += token._consumed
 						token = next_token(input, current_index)
-						if token == TOKEN_SPACE:
-							current_index += token._consumed
-							token = next_token(input, current_index)
-						arg_type = GdObjects.string_as_typeof(token._token)
+					arg_type = GdObjects.string_as_typeof(token._token)
 				elif token == TOKEN_ARGUMENT_TYPE_ASIGNMENT:
-						arg_value = _parse_end_function(input.substr(current_index), true)
-						current_index += arg_value.length()
+					arg_value = _parse_end_function(input.substr(current_index), true)
+					current_index += arg_value.length()
 				elif token == TOKEN_ARGUMENT_ASIGNMENT:
-						token = next_token(input, current_index)
-						arg_value = _parse_end_function(input.substr(current_index), true)
-						current_index += arg_value.length()
+					token = next_token(input, current_index)
+					arg_value = _parse_end_function(input.substr(current_index), true)
+					current_index += arg_value.length()
 				elif token == TOKEN_BRACKET_OPEN:
-						bracket += 1
-						# if value a function?
-						if bracket > 1:
-							# complete the argument value
-							var func_begin = input.substr(current_index-TOKEN_BRACKET_OPEN._consumed)
-							var func_body = _parse_end_function(func_begin)
-							arg_value += func_body
-							# fix parse index to end of value
-							current_index += func_body.length() - TOKEN_BRACKET_OPEN._consumed - TOKEN_BRACKET_CLOSE._consumed
+					bracket += 1
+					# if value a function?
+					if bracket > 1:
+						# complete the argument value
+						var func_begin = input.substr(current_index-TOKEN_BRACKET_OPEN._consumed)
+						var func_body = _parse_end_function(func_begin)
+						arg_value += func_body
+						# fix parse index to end of value
+						current_index += func_body.length() - TOKEN_BRACKET_OPEN._consumed - TOKEN_BRACKET_CLOSE._consumed
 				elif token == TOKEN_BRACKET_CLOSE:
-						bracket -= 1
-						# end of function
-						if bracket == 0:
-							break
+					bracket -= 1
+					# end of function
+					if bracket == 0:
+						break
 				elif token == TOKEN_ARGUMENT_SEPARATOR:
-						if bracket <= 1:
-							break
+					if bracket <= 1:
+						break
 			arg_value = arg_value.lstrip(" ")
 			if arg_type == TYPE_NIL and arg_value != GdFunctionArgument.UNDEFINED:
 				if arg_value.begins_with("Color."):
@@ -509,6 +539,24 @@ func _parse_end_function(input: String, remove_trailing_char := false) -> String
 
 	while current_index < len(input) and not end_of_func:
 		var character = input[current_index]
+		# step over strings
+		if character == "'" :
+			current_index = input.find("'", current_index+1) + 1
+			if current_index == 0:
+				push_error("Parsing error on '%s', can't evaluate end of string." % input)
+				return ""
+			continue
+		if character == '"' :
+			# test for string blocks
+			if input.find('"""', current_index) == current_index:
+				current_index = input.find('"""', current_index+3) + 3
+			else:
+				current_index = input.find('"', current_index+1) + 1
+			if current_index == 0:
+				push_error("Parsing error on '%s', can't evaluate end of string." % input)
+				return ""
+			continue
+
 		match character:
 			# count if inside an array
 			"[": in_array += 1
@@ -559,11 +607,13 @@ func extract_source_code(script_path :PackedStringArray) -> PackedStringArray:
 
 
 func extract_func_signature(rows :PackedStringArray, index :int) -> String:
-	var signature = ""
+	var signature := ""
+
 	for rowIndex in range(index, rows.size()):
-		signature += rows[rowIndex].strip_edges().replace("\t", "")
-		if is_func_end(signature):
-			return signature
+		var row := rows[rowIndex]
+		signature += row + "\n"
+		if is_func_end(row):
+			return signature.strip_edges()
 	push_error("Can't fully extract function signature of '%s'" % rows[index])
 	return ""
 

addons/gdUnit4/test/core/ParameterizedTestCaseTest.gd

@@ -174,3 +174,37 @@ func test_with_string_paramset(
 ):
 	var current := " ".join(values)
 	assert_that(current.strip_edges()).is_equal(expected)
+
+
+# https://github.com/MikeSchulze/gdUnit4/issues/213
+func test_with_string_contains_brackets(
+	test_index :int,
+	value :String,
+	test_parameters := [
+		[1, "flowchart TD\nid>This is a flag shaped node]"],
+		[2, "flowchart TD\nid(((This is a double circle node)))"],
+		[3, "flowchart TD\nid((This is a circular node))"],
+		[4, "flowchart TD\nid>This is a flag shaped node]"],
+		[5, "flowchart TD\nid{'This is a rhombus node'}"],
+		[6, 'flowchart TD\nid((This is a circular node))'],
+		[7, 'flowchart TD\nid>This is a flag shaped node]'],
+		[8, 'flowchart TD\nid{"This is a rhombus node"}'],
+		[9, """
+			flowchart TD
+			id{"This is a rhombus node"}
+			"""],
+	]
+):
+	match test_index:
+		1: assert_str(value).is_equal("flowchart TD\nid>This is a flag shaped node]")
+		2: assert_str(value).is_equal("flowchart TD\nid(((This is a double circle node)))")
+		3: assert_str(value).is_equal("flowchart TD\nid((This is a circular node))")
+		4: assert_str(value).is_equal("flowchart TD\nid>" + "This is a flag shaped node]")
+		5: assert_str(value).is_equal("flowchart TD\nid{'This is a rhombus node'}")
+		6: assert_str(value).is_equal('flowchart TD\nid((This is a circular node))')
+		7: assert_str(value).is_equal('flowchart TD\nid>This is a flag shaped node]')
+		8: assert_str(value).is_equal('flowchart TD\nid{"This is a rhombus node"}')
+		9: assert_str(value).is_equal("""
+			flowchart TD
+			id{"This is a rhombus node"}
+			""")

addons/gdUnit4/test/core/parse/GdScriptParserTest.gd

@@ -337,15 +337,38 @@ func test_extract_function_signature() -> void:
 	var rows = _parser.extract_source_code(path)
 
 	assert_that(_parser.extract_func_signature(rows, 12))\
-		.is_equal('func a1(set_name:String, path:String="", load_on_init:bool=false,set_auto_save:bool=false, set_network_sync:bool=false) -> void:')
+		.is_equal("""
+			func a1(set_name:String, path:String="", load_on_init:bool=false, 
+				set_auto_save:bool=false, set_network_sync:bool=false
+			) -> void:""".dedent().trim_prefix("\n"))
 	assert_that(_parser.extract_func_signature(rows, 19))\
-		.is_equal('func a2(set_name:String, path:String="", load_on_init:bool=false,set_auto_save:bool=false, set_network_sync:bool=false) -> void:')
+		.is_equal("""
+			func a2(set_name:String, path:String="", load_on_init:bool=false, 
+				set_auto_save:bool=false, set_network_sync:bool=false
+			) -> 	void:""".dedent().trim_prefix("\n"))
 	assert_that(_parser.extract_func_signature(rows, 26))\
-		.is_equal('func a3(set_name:String, path:String="", load_on_init:bool=false,set_auto_save:bool=false, set_network_sync:bool=false) :')
+		.is_equal("""
+			func a3(set_name:String, path:String="", load_on_init:bool=false, 
+				set_auto_save:bool=false, set_network_sync:bool=false
+			) :""".dedent().trim_prefix("\n"))
 	assert_that(_parser.extract_func_signature(rows, 33))\
-		.is_equal('func a4(set_name:String,path:String="",load_on_init:bool=false,set_auto_save:bool=false,set_network_sync:bool=false):')
+		.is_equal("""
+			func a4(set_name:String,
+				path:String="",
+				load_on_init:bool=false,
+				set_auto_save:bool=false,
+				set_network_sync:bool=false
+			):""".dedent().trim_prefix("\n"))
 	assert_that(_parser.extract_func_signature(rows, 43))\
-		.is_equal('func a5(value : Array,expected : String,test_parameters : Array = [[ ["a"], "a" ],[ ["a", "very", "long", "argument"], "a very long argument" ],]):')
+		.is_equal("""
+			func a5(
+				value : Array,
+				expected : String,
+				test_parameters : Array = [
+					[ ["a"], "a" ],
+					[ ["a", "very", "long", "argument"], "a very long argument" ],
+				]
+			):""".dedent().trim_prefix("\n"))
 
 
 func test_strip_leading_spaces():
@@ -480,17 +503,26 @@ func test_is_func_end() -> void:
 
 
 func test_extract_func_signature_multiline() -> void:
-	var source_code = [
-	"func test_parameterized(a: int, b :int, c :int, expected :int, parameters = [\n",
-	"	[1, 2, 3, 6],\n",
-	"	[3, 4, 5, 11],\n",
-	"	[6, 7, 8, 21] ]):\n",
-	"	\n",
-	"	assert_that(a+b+c).is_equal(expected)\n"
-	]
+	var source_code = """
+		
+		func test_parameterized(a: int, b :int, c :int, expected :int, parameters = [
+			[1, 2, 3, 6],
+			[3, 4, 5, 11],
+			[6, 7, 8, 21] ]):
+			
+			assert_that(a+b+c).is_equal(expected)
+		""".dedent().split("\n")
+
 	var fs = _parser.extract_func_signature(source_code, 0)
 
-	assert_that(fs).is_equal("func test_parameterized(a: int, b :int, c :int, expected :int, parameters = [[1, 2, 3, 6],[3, 4, 5, 11],[6, 7, 8, 21] ]):")
+	assert_that(fs).is_equal("""
+		func test_parameterized(a: int, b :int, c :int, expected :int, parameters = [
+			[1, 2, 3, 6],
+			[3, 4, 5, 11],
+			[6, 7, 8, 21] ]):"""
+			.dedent()
+			.trim_prefix("\n")
+		)
 
 
 func test_parse_func_description_paramized_test():