Issue #207 results export

.gutconfig.json

@@ -3,7 +3,7 @@
   "should_maximize":true,
   "should_exit":true,
   "ignore_pause":true,
-  "log_level":1,
+  "log_level":3,
   "opacity":100,
   "double_strategy":"partial",
   "include_subdirs":true,

CHANGES.md

@@ -5,7 +5,11 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 
 # NEXT RELEASE (changes to master since last release)
-
+## Features
+* __Issue 207__ Added ability to export test results in the JUnit XML format.
+  *  Added "Junit Xml File" setting to the Gut control to specify the file.  "Junit Xml Timestamp" will include an epoch timestamp in the filename.
+  *  `-gjunit_xml_file` and `-gjunit_xml_timestamp` are supported on the command line.
+  *   `junit_xml_file` and `junit_xml_timestamp` are supported in the `.gutconfig.json` file.
 ## Bug Fixes
 * __Issue 268__ Add message when `assert_signal_emitted_with_parameters` is passed bad parameters.
 * __Issue 258__ `yield_to` now supports signals with up to 9 parameters.  This is the same limit  supported by `watch_signals`.

addons/gut/gut.gd

@@ -49,6 +49,8 @@ var _double_strategy = 1  setget set_double_strategy, get_double_strategy
 var _pre_run_script = '' setget set_pre_run_script, get_pre_run_script
 var _post_run_script = '' setget set_post_run_script, get_post_run_script
 var _color_output = false setget set_color_output, get_color_output
+var _junit_xml_file = '' setget set_junit_xml_file, get_junit_xml_file
+var _junit_xml_timestamp = false setget set_junit_xml_timestamp, get_junit_xml_timestamp
 # -- End Settings --
 
 
@@ -494,15 +496,39 @@ func _end_run():
 	_is_running = false
 	update()
 	_run_hook_script(_post_run_script_instance)
+	_export_results()
 	emit_signal(SIGNAL_TESTS_FINISHED)
-	
+
 	if _utils.should_display_latest_version:
 		p("")
 		p(str("GUT version ",_utils.latest_version," is now available."))
 
 	_gui.set_title("Finished.")
 
 
+# ------------------------------------------------------------------------------
+# Add additional export types here.
+# ------------------------------------------------------------------------------
+func _export_results():
+	if(_junit_xml_file != ''):
+		_export_junit_xml()
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func _export_junit_xml():
+	var exporter = _utils.JunitExporter.new()
+	var output_file = _junit_xml_file
+
+	if(_junit_xml_timestamp):
+		var ext = "." + output_file.get_extension()
+		output_file = output_file.replace(ext, str("_", OS.get_unix_time(), ext))
+
+	var f_result = exporter.write_file(self, output_file)
+	if(f_result == OK):
+		p(str("Results saved to ", output_file))
+
+
+
 # ------------------------------------------------------------------------------
 # Checks the passed in thing to see if it is a "function state" object that gets
 # returned when a function yields.
@@ -1576,3 +1602,25 @@ func show_orphans(should):
 # ------------------------------------------------------------------------------
 func get_autofree():
 	return _autofree
+
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func get_junit_xml_file():
+	return _junit_xml_file
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func set_junit_xml_file(junit_xml_file):
+	_junit_xml_file = junit_xml_file
+
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func get_junit_xml_timestamp():
+	return _junit_xml_timestamp
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+func set_junit_xml_timestamp(junit_xml_timestamp):
+	_junit_xml_timestamp = junit_xml_timestamp

addons/gut/gut_cmdln.gd

@@ -135,6 +135,8 @@ var options = {
 	ignore_pause = false,
 	include_subdirs = false,
 	inner_class = '',
+	junit_xml_file = '',
+	junit_xml_timestamp = false,
 	log_level = 1,
 	opacity = 100,
 	post_run_script = '',
@@ -195,6 +197,9 @@ func setup_options():
 	opts.add('-gfont_size', options.font_size, 'Font size, default "[default]"')
 	opts.add('-gbackground_color', options.background_color, 'Background color as an html color, default "[default]"')
 	opts.add('-gfont_color',options.font_color, 'Font color as an html color, default "[default]"')
+
+	opts.add('-gjunit_xml_file', options.junit_xml_file, 'Export results of run to this file in the Junit XML format.')
+	opts.add('-gjunit_xml_timestamp', options.junit_xml_timestamp, 'Include a timestamp in the -gjunit_xml_file, default [default]')
 	return opts
 
 
@@ -227,6 +232,9 @@ func extract_command_line_options(from, to):
 	to.background_color = from.get_value('-gbackground_color')
 	to.font_color = from.get_value('-gfont_color')
 
+	to.junit_xml_file = from.get_value('-gjunit_xml_file')
+	to.junit_xml_timestamp = from.get_value('-gjunit_xml_timestamp')
+
 
 func load_options_from_config_file(file_path, into):
 	# SHORTCIRCUIT
@@ -299,6 +307,8 @@ func apply_options(opts):
 	_tester.set_post_run_script(opts.post_run_script)
 	_tester.set_color_output(!opts.disable_colors)
 	_tester.show_orphans(!opts.hide_orphans)
+	_tester.set_junit_xml_file(opts.junit_xml_file)
+	_tester.set_junit_xml_timestamp(opts.junit_xml_timestamp)
 
 	_tester.get_gui().set_font_size(opts.font_size)
 	_tester.get_gui().set_font(opts.font_name)

addons/gut/hook_script.gd

@@ -1,7 +1,10 @@
 # ------------------------------------------------------------------------------
 # This script is the base for custom scripts to be used in pre and post
 # run hooks.
+#
+# To use, inherit from this script and then implement the run method.
 # ------------------------------------------------------------------------------
+var JunitXmlExport = load('res://addons/gut/junit_xml_export.gd')
 
 # This is the instance of GUT that is running the tests.  You can get
 # information about the run from this object.  This is set by GUT when the
@@ -12,6 +15,7 @@ var gut  = null
 var _exit_code = null
 
 var _should_abort =  false
+
 # Virtual method that will be called by GUT after instantiating
 # this script.
 func run():

addons/gut/junit_xml_export.gd

@@ -0,0 +1,94 @@
+# ------------------------------------------------------------------------------
+# Creates an export of a test run in the JUnit XML format.
+# ------------------------------------------------------------------------------
+var _utils = load('res://addons/gut/utils.gd').get_instance()
+
+var _exporter = _utils.ResultExporter.new()
+
+func indent(s, ind):
+	var to_return = ind + s
+	to_return = to_return.replace("\n", "\n" + ind)
+	return to_return
+
+
+func add_attr(name, value):
+	return str(name, '="', value, '" ')
+
+func _export_test_result(test):
+	var to_return = ''
+
+	# Right now the pending and failure messages won't fit in the message
+	# attribute because they can span multiple lines and need to be escaped.
+	if(test.status == 'pending'):
+		var skip_tag = str("<skipped message=\"pending\">", test.pending[0], "</skipped>")
+		to_return += skip_tag
+	elif(test.status == 'fail'):
+		var fail_tag = str("<failure message=\"failed\">", test.failing[0], "</failure>")
+		to_return += fail_tag
+
+	return to_return
+
+
+func _export_tests(script_result, classname):
+	var to_return = ""
+
+	for key in script_result.keys():
+		var test = script_result[key]
+		var assert_count = test.passing.size() + test.failing.size()
+		to_return += "<testcase "
+		to_return += add_attr("name", key)
+		to_return += add_attr("assertions", assert_count)
+		to_return += add_attr("status", test.status)
+		to_return += add_attr("classname", classname)
+		to_return += ">\n"
+
+		to_return += _export_test_result(test)
+
+		to_return += "</testcase>\n"
+
+	return to_return
+
+
+func _export_scripts(exp_results):
+	var to_return = ""
+	for key in exp_results.test_scripts.scripts.keys():
+		var s = exp_results.test_scripts.scripts[key]
+		to_return += "<testsuite "
+		to_return += add_attr("name", key)
+		to_return += add_attr("tests", s.props.tests)
+		to_return += add_attr("failures", s.props.failures)
+		to_return += add_attr("skipped", s.props.pending)
+		to_return += ">\n"
+
+		to_return += indent(_export_tests(s.tests, key), "    ")
+
+		to_return += "</testsuite>\n"
+
+	return to_return
+
+
+func get_results_xml(gut):
+	var exp_results = _exporter.get_results_dictionary(gut)
+	var to_return = '<?xml version="1.0" encoding="UTF-8"?>' + "\n"
+	to_return += '<testsuites '
+	to_return += add_attr("name", 'GutTests')
+	to_return += add_attr("failures", exp_results.test_scripts.props.failures)
+	to_return += add_attr('tests', exp_results.test_scripts.props.tests)
+	to_return += ">\n"
+
+	to_return += indent(_export_scripts(exp_results), "  ")
+
+	to_return += '</testsuites>'
+	return to_return
+
+
+func write_file(gut, path):
+	var xml = get_results_xml(gut)
+
+	var f_result = _utils.write_file(path, xml)
+	if(f_result != OK):
+		var msg = str("Error:  ", f_result, ".  Could not create export file ", path)
+		_utils.get_logger().error(msg)
+
+	return f_result
+

addons/gut/plugin_control.gd

@@ -104,6 +104,10 @@ export(int, 'FULL', 'PARTIAL') var _double_strategy = 1
 export(String, FILE) var _pre_run_script = ''
 # Path and filename to the script to run after all tests are run.
 export(String, FILE) var _post_run_script = ''
+# Path to the file that gut will export results to in the junit xml format
+export(String, FILE) var _junit_xml_file = ''
+# Flag to include a timestamp in the filename of _junit_xml_file
+export(bool) var _junit_xml_timestamp = false
 # ------------------------------------------------------------------------------
 
 
@@ -193,6 +197,8 @@ func _setup_gut():
 	_gut.set_post_run_script(_post_run_script)
 	_gut.set_color_output(_color_output)
 	_gut.show_orphans(_show_orphans)
+	_gut.set_junit_xml_file(_junit_xml_file)
+	_gut.set_junit_xml_timestamp(_junit_xml_timestamp)
 
 	get_parent().add_child(_gut)
 

addons/gut/result_exporter.gd

@@ -0,0 +1,73 @@
+# ------------------------------------------------------------------------------
+# Creates a structure that contains all the data about the results of running
+# tests.  This was created to make an intermediate step organizing the result
+# of a run and exporting it in a specific format.  This can also serve as a
+# unofficial GUT export format.
+# ------------------------------------------------------------------------------
+var _utils = load('res://addons/gut/utils.gd').get_instance()
+
+func _export_tests(summary_script):
+	var to_return = {}
+	var tests = summary_script.get_tests()
+	for key in tests.keys():
+		to_return[key] = {
+			"status":tests[key].get_status(),
+			"passing":tests[key].pass_texts,
+			"failing":tests[key].fail_texts,
+			"pending":tests[key].pending_texts
+		}
+
+	return to_return
+
+# TODO
+#	errors
+func _export_scripts(summary):
+	if(summary == null):
+		return {}
+
+	var scripts = {}
+
+	for s in summary.get_scripts():
+		scripts[s.name] = {
+			'props':{
+				"tests":s._tests.size(),
+				"pending":s.get_pending_count(),
+				"failures":s.get_fail_count(),
+			},
+			"tests":_export_tests(s)
+		}
+	return scripts
+
+
+# TODO
+#	time
+#	errors
+func get_results_dictionary(gut):
+	var summary = gut.get_summary()
+	var scripts = _export_scripts(summary)
+	var totals = summary.get_totals()
+
+	var result =  {
+		'test_scripts':{
+			"props":{
+				"pending":totals.pending,
+				"failures":totals.failing,
+				"tests":totals.tests,
+			},
+			"scripts":scripts
+		}
+	}
+	return result
+
+
+func write_json_file(gut, path):
+	var dict = get_results_dictionary(gut)
+	var json = JSON.print(dict, ' ')
+
+	var f_result = _utils.write_file(path, json)
+	if(f_result != OK):
+		var msg = str("Error:  ", f_result, ".  Could not create export file ", path)
+		_utils.get_logger().error(msg)
+
+	return f_result
+

addons/gut/summary.gd

@@ -18,6 +18,17 @@ class Test:
 			to_return += str(pad, '[Pending]:  ', pending_texts[i], "\n")
 		return to_return
 
+	func get_status():
+		var to_return = 'no asserts'
+		if(pending_texts.size() > 0):
+			to_return = 'pending'
+		elif(fail_texts.size() > 0):
+			to_return = 'fail'
+		elif(pass_texts.size() > 0):
+			to_return = 'pass'
+
+		return to_return
+
 # ------------------------------------------------------------------------------
 # Contains all the results for a single test-script/inner class.  Persists the
 # names of the tests and results and the order in which  the tests were run.
@@ -66,12 +77,15 @@ class TestScript:
 		var t = get_test_obj(test_name)
 		t.pending_texts.append(reason)
 
+	func get_tests():
+		return _tests
+
 # ------------------------------------------------------------------------------
 # Summary Class
 #
 # This class holds the results of all the test scripts and Inner Classes that
 # were run.
-# -------------------------------------------d-----------------------------------
+# ------------------------------------------------------------------------------
 var _scripts = []
 
 func add_script(name):