Godot 4 errors as failures (#502)

* errors cause failures, flag defaulted to true
* errors_cause_failure options added to gutconfig, cli, and panel
* changes
* removed a test test
* tweaked output for parameterized tests
* fixed issue with panel tree being empty when all passed.  a couple other tweaks

.gutconfig.json

@@ -9,7 +9,7 @@
   "should_maximize": false,
   "should_exit": true,
   "ignore_pause": true,
-  "log_level": 0,
+  "log_level": 1,
   "double_strategy": "INCLUDE_NATIVE",
   "inner_class": "",
   "disable_colors": false,

CHANGES.md

@@ -4,7 +4,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 
 # 9.1.0
-* Tested and verified in 4.1
+* GUT generated errors now cause tests to fail (not engine errors, just things GUT thinks are bad).  You can disable this through the CLI, .gutconfig, or the panel.
 * Changes to Double Strategy and Double/Partial Double creation to fix #482.
     * See [Double-Strategy](https://bitwes.github.io/GutWiki/Godot4/Double-Strategy.html) in the wiki for more information.
     * The default strategy has been changed back to `SCRIPT_ONLY` (a bug caused it to change).  Due to how the Godot Engine calls native methods, the overrides may not be called by the engine so spying and stubbing may not work in some scenarios.

addons/gut/gui/ResultsTree.gd

@@ -48,8 +48,8 @@ func _ready():
 
 func _test_running_setup():
 	load_json_file('user://.gut_editor.json')
-	
-	
+
+
 func _on_tree_item_selected():
 	var item = _ctrls.tree.get_selected()
 	var item_meta = item.get_metadata(0)
@@ -263,7 +263,7 @@ func _free_childless_scripts():
 
 
 func _show_all_passed():
-	if(_root.get_children() == null):
+	if(_root.get_children().size() == 0):
 		add_centered_text('Everything passed!')
 
 

addons/gut/gui/gut_config_gui.gd

@@ -354,6 +354,8 @@ func set_options(options):
 		'This is the default, you can override this at the script level or when creating doubles.')
 	_cfg_ctrls['double_strategy'].selected = GutUtils.get_enum_value(
 		options.double_strategy, GutUtils.DOUBLE_STRATEGY, GutUtils.DOUBLE_STRATEGY.SCRIPT_ONLY)
+	_add_boolean('errors_cause_failure', !options.errors_do_not_cause_failure, 'Errors cause failures.',
+		"When GUT generates an error (not an engine error) it causes tests to fail.")
 
 
 	_add_title("Panel Output") # ----------------------------------
@@ -425,6 +427,7 @@ func set_options(options):
 
 	print('GUT config loaded')
 
+
 func get_options(base_opts):
 	var to_return = base_opts.duplicate()
 
@@ -435,6 +438,7 @@ func get_options(base_opts):
 	to_return.should_exit = _cfg_ctrls.should_exit.button_pressed
 	to_return.should_exit_on_success = _cfg_ctrls.should_exit_on_success.button_pressed
 	to_return.double_strategy = _cfg_ctrls.double_strategy.selected
+	to_return.errors_do_not_cause_failure = !_cfg_ctrls.errors_cause_failure.button_pressed
 
 	#Output
 	to_return.panel_options.font_name = _cfg_ctrls.output_font_name.get_item_text(

addons/gut/gut.gd

@@ -150,10 +150,6 @@ var unit_test_name = _unit_test_name :
 	get: return _unit_test_name
 	set(val): _unit_test_name = val
 
-# ###########################
-# Public Properties
-# ###########################
-
 var _parameter_handler = null
 # This is populated by test.gd each time a paramterized test is encountered
 # for the first time.
@@ -183,6 +179,11 @@ var add_children_to = self :
 	set(val): _add_children_to = val
 
 
+var _treat_error_as_failure = true
+var treat_error_as_failure = _treat_error_as_failure:
+	get: return _treat_error_as_failure
+	set(val): _treat_error_as_failure = val
+
 # ------------
 # Read only
 # ------------
@@ -249,6 +250,7 @@ var _done = false
 # msecs ticks when run was started
 var _start_time = 0.0
 
+# Collected Test instance for the current test being run.
 var _current_test = null
 var _pause_before_teardown = false
 
@@ -494,34 +496,12 @@ func _export_junit_xml():
 		p(str("Results saved to ", output_file))
 
 
-
-# ------------------------------------------------------------------------------
-# Checks the passed in thing to see if it is a "function state" object that gets
-# returned when a function yields.
-# ------------------------------------------------------------------------------
-func _is_function_state(script_result):
-	return false
-	# TODO 4.0 Keep this until we know how they are going to handle the
-	# 4.0 equivalent of GDScriptFunctionState
-	# return script_result != null and \
-	# 	typeof(script_result) == TYPE_OBJECT and \
-	# 	script_result is GDScriptFunctionState and \
-	# 	script_result.is_valid()
-
 # ------------------------------------------------------------------------------
 # Print out the heading for a new script
 # ------------------------------------------------------------------------------
-func _print_script_heading(script):
-	if(_does_class_name_match(_inner_class_name, script.inner_class_name)):
-		var fmt = _lgr.fmts.underline
-		var divider = '-----------------------------------------'
-
-		var text = ''
-		if(script.inner_class_name == null):
-			text = script.path
-		else:
-			text = str(script.path, '.', script.inner_class_name)
-		_lgr.log("\n\n" + text, fmt)
+func _print_script_heading(coll_script):
+	if(_does_class_name_match(_inner_class_name, coll_script.inner_class_name)):
+		_lgr.log(str("\n\n", coll_script.get_full_name()), _lgr.fmts.underline)
 
 
 # ------------------------------------------------------------------------------
@@ -531,6 +511,7 @@ func _does_class_name_match(the_class_name, script_class_name):
 	return (the_class_name == null or the_class_name == '') or \
 		(script_class_name != null and str(script_class_name).findn(the_class_name) != -1)
 
+
 # ------------------------------------------------------------------------------
 # ------------------------------------------------------------------------------
 func _setup_script(test_script):
@@ -558,6 +539,7 @@ func _get_indexes_matching_script_name(name):
 			indexes.append(i)
 	return indexes
 
+
 # ------------------------------------------------------------------------------
 # ------------------------------------------------------------------------------
 func _get_indexes_matching_path(path):
@@ -567,6 +549,7 @@ func _get_indexes_matching_path(path):
 			indexes.append(i)
 	return indexes
 
+
 # ------------------------------------------------------------------------------
 # Execute all calls of a parameterized test.
 # ------------------------------------------------------------------------------
@@ -628,6 +611,7 @@ func _run_test(script_inst, test_name):
 
 	_doubler.get_ignored_methods().clear()
 
+
 # ------------------------------------------------------------------------------
 # Calls after_all on the passed in test script and takes care of settings so all
 # logger output appears indented and with a proper heading
@@ -652,6 +636,7 @@ func _call_before_all(test_script, collected_script):
 
 	_current_test = null
 
+
 # ------------------------------------------------------------------------------
 # Calls after_all on the passed in test script and takes care of settings so all
 # logger output appears indented and with a proper heading
@@ -675,6 +660,7 @@ func _call_after_all(test_script, collected_script):
 
 	_current_test = null
 
+
 # ------------------------------------------------------------------------------
 # Run all tests in a script.  This is the core logic for running tests.
 # ------------------------------------------------------------------------------
@@ -824,9 +810,21 @@ func _test_the_scripts(indexes=[]):
 func _pass(text=''):
 	if(_current_test):
 		_current_test.add_pass(text)
-	else:
-		# I don't think this is valid anymore.
-		_lgr.error('Encountered a call to _pass when there is no current test')
+
+
+# ------------------------------------------------------------------------------
+# Returns an empty string or "(call #x) " if the current test being run has
+# parameters.  The
+# ------------------------------------------------------------------------------
+func get_call_count_text():
+	var to_return = ''
+	if(_parameter_handler != null):
+		# This uses get_call_count -1 because test.gd's use_parameters method
+		# should have been called before we get to any calls for this method
+		# just due to how use_parameters works.  There isn't a way to know
+		# whether we are before or after that call.
+		to_return = str('params[', _parameter_handler.get_call_count() -1, '] ')
+	return to_return
 
 
 # ------------------------------------------------------------------------------
@@ -838,14 +836,26 @@ func _fail(text=''):
 		p(line_text, LOG_LEVEL_FAIL_ONLY)
 		# format for summary
 		line_text =  "\n    " + line_text
-		var call_count_text = ''
-		if(_parameter_handler != null):
-			call_count_text = str('(call #', _parameter_handler.get_call_count(), ') ')
+		var call_count_text = get_call_count_text()
 		_current_test.line_number = line_number
 		_current_test.add_fail(call_count_text + text + line_text)
-	else:
-		# I don't think this is valid anymore.
-		_lgr.error('Encountered a call to _fail when there is no current test')
+
+
+# ------------------------------------------------------------------------------
+# This is "private" but is only used by the logger, it is not used internally.
+# It was either, make this weird method or "do it the right way" with signals
+# or some other crazy mechanism.
+# ------------------------------------------------------------------------------
+func _fail_for_error(err_text):
+	if(_current_test != null and treat_error_as_failure):
+		_fail(err_text)
+
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func _pending(text=''):
+	if(_current_test):
+		_current_test.add_pending(text)
 
 
 # ------------------------------------------------------------------------------
@@ -864,13 +874,6 @@ func _extract_line_number(current_test):
 	return line_number
 
 
-# ------------------------------------------------------------------------------
-# ------------------------------------------------------------------------------
-func _pending(text=''):
-	if(_current_test):
-		_current_test.add_pending(text)
-
-
 # ------------------------------------------------------------------------------
 # Gets all the files in a directory and all subdirectories if include_subdirectories
 # is true.  The files returned are all sorted by name.

addons/gut/gut_cmdln.gd

@@ -126,50 +126,68 @@ var _final_opts = []
 
 func setup_options(options, font_names):
 	var opts = Optparse.new()
-	opts.set_banner(('This is the command line interface for the unit testing tool Gut.  With this ' +
-					'interface you can run one or more test scripts from the command line.  In order ' +
-					'for the Gut options to not clash with any other godot options, each option starts ' +
-					'with a "g".  Also, any option that requires a value will take the form of ' +
-					'"-g<name>=<value>".  There cannot be any spaces between the option, the "=", or ' +
-					'inside a specified value or godot will think you are trying to run a scene.'))
-
-	opts.add('-gtest', [], 'Comma delimited list of full paths to test scripts to run.')
+	opts.set_banner(
+"""
+The GUT CLI
+-----------
+The default behavior for GUT is to load options from a res://.gutconfig.json if
+it exists.  Any options specified on the command line will take precedence over
+options specified in the gutconfig file.  You can specify a different gutconfig
+file with the -gconfig option.
+
+To generate a .gutconfig.json file you can use -gprint_gutconfig_sample
+To see the effective values of a CLI command and a gutconfig use -gpo
+
+Any option that requires a value will take the form of \"-g<name>=<value>\".
+There cannot be any spaces between the option, the \"=\", or ' + 'inside a
+specified value or godot will think you are trying to run a scene.
+""")
+	# Run specific things
+	opts.add('-gselect', '', ('All scripts that contain the specified string in their filename will be ran'))
+	opts.add('-ginner_class', '', 'Only run inner classes that contain the specified string int their name.')
+	opts.add('-gunit_test_name', '', ('Any test that contains the specified text will be run, all others will be skipped.'))
+
+	# Run Config
+	opts.add('-ginclude_subdirs', false, 'Include subdirectories of -gdir.')
 	opts.add('-gdir', options.dirs, 'Comma delimited list of directories to add tests from.')
+	opts.add('-gtest', [], 'Comma delimited list of full paths to test scripts to run.')
 	opts.add('-gprefix', options.prefix, 'Prefix used to find tests when specifying -gdir.  Default "[default]".')
 	opts.add('-gsuffix', options.suffix, 'Test script suffix, including .gd extension.  Default "[default]".')
+	opts.add('-gconfig', 'res://.gutconfig.json', 'A config file that contains configuration information.  Default is res://.gutconfig.json')
+	opts.add('-gpre_run_script', '', 'pre-run hook script path')
+	opts.add('-gpost_run_script', '', 'post-run hook script path')
+	opts.add('-gerrors_do_not_cause_failure', false, 'When an internal GUT error occurs tests will fail.  With this option set, that does not happen.')
+	opts.add('-gdouble_strategy', 'SCRIPT_ONLY', 'Default strategy to use when doubling.  Valid values are [INCLUDE_NATIVE, SCRIPT_ONLY].  Default "[default]"')
+
+	# Misc
+	opts.add('-gpaint_after', options.paint_after, 'Delay before GUT will add a 1 frame pause to paint the screen/GUI.  default [default]')
+
+	# Display options
+	opts.add('-glog', options.log_level, 'Log level.  Default [default]')
 	opts.add('-ghide_orphans', false, 'Display orphan counts for tests and scripts.  Default "[default]".')
 	opts.add('-gmaximize', false, 'Maximizes test runner window to fit the viewport.')
 	opts.add('-gcompact_mode', false, 'The runner will be in compact mode.  This overrides -gmaximize.')
-	opts.add('-gexit', false, 'Exit after running tests.  If not specified you have to manually close the window.')
-	opts.add('-gexit_on_success', false, 'Only exit if all tests pass.')
-	opts.add('-glog', options.log_level, 'Log level.  Default [default]')
-	opts.add('-gignore_pause', false, 'Ignores any calls to gut.pause_before_teardown.')
-	opts.add('-gselect', '', ('Select a script to run initially.  The first script that ' +
-							'was loaded using -gtest or -gdir that contains the specified ' +
-							'string will be executed.  You may run others by interacting ' +
-							'with the GUI.'))
-	opts.add('-gunit_test_name', '', ('Name of a test to run.  Any test that contains the specified ' +
-								'text will be run, all others will be skipped.'))
-	opts.add('-gh', false, 'Print this help, then quit')
-	opts.add('-gconfig', 'res://.gutconfig.json', 'A config file that contains configuration information.  Default is res://.gutconfig.json')
-	opts.add('-ginner_class', '', 'Only run inner classes that contain this string')
 	opts.add('-gopacity', options.opacity, 'Set opacity of test runner window. Use range 0 - 100. 0 = transparent, 100 = opaque.')
-	opts.add('-gpo', false, 'Print option values from all sources and the value used, then quit.')
-	opts.add('-ginclude_subdirs', false, 'Include subdirectories of -gdir.')
-	opts.add('-gdouble_strategy', 'SCRIPT_ONLY', 'Default strategy to use when doubling.  Valid values are [INCLUDE_NATIVE, SCRIPT_ONLY].  Default "[default]"')
 	opts.add('-gdisable_colors', false, 'Disable command line colors.')
-	opts.add('-gpre_run_script', '', 'pre-run hook script path')
-	opts.add('-gpost_run_script', '', 'post-run hook script path')
-	opts.add('-gprint_gutconfig_sample', false, 'Print out json that can be used to make a gutconfig file then quit.')
-
 	opts.add('-gfont_name', options.font_name, str('Valid values are:  ', font_names, '.  Default "[default]"'))
 	opts.add('-gfont_size', options.font_size, 'Font size, default "[default]"')
 	opts.add('-gbackground_color', options.background_color, 'Background color as an html color, default "[default]"')
 	opts.add('-gfont_color',options.font_color, 'Font color as an html color, default "[default]"')
-	opts.add('-gpaint_after', options.paint_after, 'Delay before GUT will add a 1 frame pause to paint the screen/GUI.  default [default]')
 
+	# End Behavior
+	opts.add('-gexit', false, 'Exit after running tests.  If not specified you have to manually close the window.')
+	opts.add('-gexit_on_success', false, 'Only exit if all tests pass.')
+	opts.add('-gignore_pause', false, 'Ignores any calls to gut.pause_before_teardown.')
+
+	# Helpish options
+	opts.add('-gh', false, 'Print this help.  You did this to see this, so you probably understand.')
+	opts.add('-gpo', false, 'Print option values from all sources and the value used.')
+	opts.add('-gprint_gutconfig_sample', false, 'Print out json that can be used to make a gutconfig file.')
+
+	# Output options
 	opts.add('-gjunit_xml_file', options.junit_xml_file, 'Export results of run to this file in the Junit XML format.')
 	opts.add('-gjunit_xml_timestamp', options.junit_xml_timestamp, 'Include a timestamp in the -gjunit_xml_file, default [default]')
+
 	return opts
 
 
@@ -195,6 +213,7 @@ func extract_command_line_options(from, to):
 	to.compact_mode = from.get_value('-gcompact_mode')
 	to.hide_orphans = from.get_value('-ghide_orphans')
 	to.suffix = from.get_value('-gsuffix')
+	to.errors_do_not_cause_failure = from.get_value('-gerrors_do_not_cause_failure')
 	to.tests = from.get_value('-gtest')
 	to.unit_test_name = from.get_value('-gunit_test_name')