From db36b7ac530c9a22f492609e2b034ad2849df15e Mon Sep 17 00:00:00 2001 From: David Bensoussan Date: Wed, 16 Jan 2019 16:34:44 +0100 Subject: [PATCH 1/5] feat: output result as junit --- openhtf/output/callbacks/junit.py | 83 +++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 openhtf/output/callbacks/junit.py diff --git a/openhtf/output/callbacks/junit.py b/openhtf/output/callbacks/junit.py new file mode 100644 index 000000000..e1ab4bbc0 --- /dev/null +++ b/openhtf/output/callbacks/junit.py @@ -0,0 +1,83 @@ +"""Module for outputting test record to JUNIT-formatted files.""" + +import base64 +from junit_xml import TestSuite, TestCase + +from openhtf.output import callbacks +from openhtf.util import data +import six + + +class OutputToJUNIT(callbacks.OutputToFile): + """Return an output callback that writes JSON Test Records. + Example filename_patterns might be: + '/data/test_records/{dut_id}.{metadata[test_name]}.json', indent=4)) or + '/data/test_records/%(dut_id)s.%(start_time_millis)s' + To use this output mechanism: + test = openhtf.Test(PhaseOne, PhaseTwo) + test.add_output_callback(openhtf.output.callbacks.OutputToJson( + '/data/test_records/{dut_id}.{metadata[test_name]}.json')) + + Args: + filename_pattern: A format string specifying the filename to write to, + will be formatted with the Test Record as a dictionary. May also be a + file-like object to write to directly. + inline_attachments: Whether attachments should be included inline in the + output. Set to False if you expect to have large binary attachments. If + True (the default), then attachments are base64 encoded to allow for + binary data that's not supported by JSON directly. + """ + + def __init__(self, filename_pattern=None, inline_attachments=True, **kwargs): + super(OutputToJUNIT, self).__init__(filename_pattern) + self.inline_attachments = inline_attachments + + # Conform strictly to the JSON spec by default. + kwargs.setdefault('allow_nan', False) + self.allow_nan = kwargs['allow_nan'] + self.test_cases = [] + + def serialize_test_record(self, test_record): + dict_test_record = self.convert_to_dict(test_record) + + for phase in dict_test_record["phases"]: + output = [] + for _, phase_data in phase["measurements"].items(): + + output.extend(["name: " + phase_data["name"], + "validators: " + str(phase_data["validators"]), + "measured_value: " + str(phase_data["measured_value"]), + "outcome: " + phase_data["outcome"], "\n"]) + + if phase["outcome"] == "PASS": + self.test_cases.append( + TestCase(phase["name"], + dict_test_record["dut_id"] + "." + + dict_test_record["metadata"]["test_name"], + (phase["end_time_millis"] - + phase["start_time_millis"]) * 1000, + "\n".join(output), + '')) + else: + self.test_cases.append( + TestCase(phase["name"], + dict_test_record["dut_id"] + "." + + dict_test_record["metadata"]["test_name"], + (phase["end_time_millis"] - + phase["start_time_millis"]) * 1000, + "\n".join(output), + '')) + + return TestSuite.to_xml_string([TestSuite("Test", self.test_cases)]) + + def convert_to_dict(self, test_record): + as_dict = data.convert_to_base_types(test_record, + json_safe=(not self.allow_nan)) + + if self.inline_attachments: + for phase, original_phase in zip(as_dict['phases'], test_record.phases): + for name, attachment in six.iteritems(phase['attachments']): + original_data = original_phase.attachments[name].data + attachment['data'] = base64.standard_b64encode( + original_data).decode('utf-8') + return as_dict From ae5d5cd97c8387c1969aa0efe8c9d8ad8cca722c Mon Sep 17 00:00:00 2001 From: David Bensoussan Date: Wed, 16 Jan 2019 16:56:00 +0100 Subject: [PATCH 2/5] cleanup: remove json unneeded references --- openhtf/output/callbacks/junit.py | 39 +++++++++++++------------------ 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/openhtf/output/callbacks/junit.py b/openhtf/output/callbacks/junit.py index e1ab4bbc0..f755dd983 100644 --- a/openhtf/output/callbacks/junit.py +++ b/openhtf/output/callbacks/junit.py @@ -1,22 +1,22 @@ """Module for outputting test record to JUNIT-formatted files.""" import base64 +import six from junit_xml import TestSuite, TestCase from openhtf.output import callbacks from openhtf.util import data -import six class OutputToJUNIT(callbacks.OutputToFile): - """Return an output callback that writes JSON Test Records. + """Return an output callback that writes JUNIT Test Records. Example filename_patterns might be: - '/data/test_records/{dut_id}.{metadata[test_name]}.json', indent=4)) or + '/data/test_records/{dut_id}.{metadata[test_name]}.xml', indent=4)) or '/data/test_records/%(dut_id)s.%(start_time_millis)s' To use this output mechanism: test = openhtf.Test(PhaseOne, PhaseTwo) - test.add_output_callback(openhtf.output.callbacks.OutputToJson( - '/data/test_records/{dut_id}.{metadata[test_name]}.json')) + test.add_output_callback(openhtf.output.callbacks.OutputToJUNIT( + '/data/test_records/{dut_id}.{metadata[test_name]}.xml')) Args: filename_pattern: A format string specifying the filename to write to, @@ -25,16 +25,11 @@ class OutputToJUNIT(callbacks.OutputToFile): inline_attachments: Whether attachments should be included inline in the output. Set to False if you expect to have large binary attachments. If True (the default), then attachments are base64 encoded to allow for - binary data that's not supported by JSON directly. + binary data that's not supported by JUNIT directly. """ - def __init__(self, filename_pattern=None, inline_attachments=True, **kwargs): + def __init__(self, filename_pattern=None, **kwargs): super(OutputToJUNIT, self).__init__(filename_pattern) - self.inline_attachments = inline_attachments - - # Conform strictly to the JSON spec by default. - kwargs.setdefault('allow_nan', False) - self.allow_nan = kwargs['allow_nan'] self.test_cases = [] def serialize_test_record(self, test_record): @@ -55,7 +50,7 @@ def serialize_test_record(self, test_record): dict_test_record["dut_id"] + "." + dict_test_record["metadata"]["test_name"], (phase["end_time_millis"] - - phase["start_time_millis"]) * 1000, + phase["start_time_millis"]) / 1000, "\n".join(output), '')) else: @@ -64,20 +59,18 @@ def serialize_test_record(self, test_record): dict_test_record["dut_id"] + "." + dict_test_record["metadata"]["test_name"], (phase["end_time_millis"] - - phase["start_time_millis"]) * 1000, + phase["start_time_millis"]) / 1000, "\n".join(output), '')) - return TestSuite.to_xml_string([TestSuite("Test", self.test_cases)]) + return TestSuite.to_xml_string([TestSuite("test", self.test_cases)]) def convert_to_dict(self, test_record): - as_dict = data.convert_to_base_types(test_record, - json_safe=(not self.allow_nan)) + as_dict = data.convert_to_base_types(test_record) - if self.inline_attachments: - for phase, original_phase in zip(as_dict['phases'], test_record.phases): - for name, attachment in six.iteritems(phase['attachments']): - original_data = original_phase.attachments[name].data - attachment['data'] = base64.standard_b64encode( - original_data).decode('utf-8') + for phase, original_phase in zip(as_dict['phases'], test_record.phases): + for name, attachment in six.iteritems(phase['attachments']): + original_data = original_phase.attachments[name].data + attachment['data'] = base64.standard_b64encode( + original_data).decode('utf-8') return as_dict From cbc93a938713786bb4211428eaf8e9f2cc346823 Mon Sep 17 00:00:00 2001 From: David Bensoussan Date: Wed, 16 Jan 2019 17:34:02 +0100 Subject: [PATCH 3/5] fix: handle phase failure --- openhtf/output/callbacks/junit.py | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/openhtf/output/callbacks/junit.py b/openhtf/output/callbacks/junit.py index f755dd983..4a91042eb 100644 --- a/openhtf/output/callbacks/junit.py +++ b/openhtf/output/callbacks/junit.py @@ -34,6 +34,7 @@ def __init__(self, filename_pattern=None, **kwargs): def serialize_test_record(self, test_record): dict_test_record = self.convert_to_dict(test_record) + test_case = None for phase in dict_test_record["phases"]: output = [] @@ -45,23 +46,25 @@ def serialize_test_record(self, test_record): "outcome: " + phase_data["outcome"], "\n"]) if phase["outcome"] == "PASS": - self.test_cases.append( - TestCase(phase["name"], - dict_test_record["dut_id"] + "." + - dict_test_record["metadata"]["test_name"], - (phase["end_time_millis"] - - phase["start_time_millis"]) / 1000, - "\n".join(output), - '')) + test_case = TestCase(phase["name"], + dict_test_record["dut_id"] + "." + + dict_test_record["metadata"]["test_name"], + (phase["end_time_millis"] - + phase["start_time_millis"]) / 1000, + "\n".join(output), + '') else: - self.test_cases.append( - TestCase(phase["name"], - dict_test_record["dut_id"] + "." + - dict_test_record["metadata"]["test_name"], - (phase["end_time_millis"] - - phase["start_time_millis"]) / 1000, - "\n".join(output), - '')) + test_case = TestCase(phase["name"], + dict_test_record["dut_id"] + "." + + dict_test_record["metadata"]["test_name"], + (phase["end_time_millis"] - + phase["start_time_millis"]) / 1000, + '', + "\n".join(output)) + + test_case.add_failure_info("failure message") + + self.test_cases.append(test_case) return TestSuite.to_xml_string([TestSuite("test", self.test_cases)]) From a03f1278d4ce8a9cf3adbc1f25d5bfc152735107 Mon Sep 17 00:00:00 2001 From: David Bensoussan Date: Wed, 16 Jan 2019 17:52:30 +0100 Subject: [PATCH 4/5] fix: add junit-xml as depend --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0e3dfa35c..bc7e2290e 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ def run(self): 'colorama>=0.3.9,<1.0', 'contextlib2>=0.5.1,<1.0', 'future>=0.16.0', + 'junit-xml>=1.8.0', 'mutablerecords>=0.4.1,<2.0', 'oauth2client>=1.5.2,<2.0', 'protobuf>=3.0.0,<4.0', From 6ab93fcff8e8fbedf081863a062daa89959bd2ee Mon Sep 17 00:00:00 2001 From: David Bensoussan Date: Mon, 21 Jan 2019 16:58:24 +0100 Subject: [PATCH 5/5] feat: add logs in output --- openhtf/output/callbacks/junit.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openhtf/output/callbacks/junit.py b/openhtf/output/callbacks/junit.py index 4a91042eb..a6cb2615d 100644 --- a/openhtf/output/callbacks/junit.py +++ b/openhtf/output/callbacks/junit.py @@ -39,11 +39,17 @@ def serialize_test_record(self, test_record): for phase in dict_test_record["phases"]: output = [] for _, phase_data in phase["measurements"].items(): + log_output = [] + + for log_data in dict_test_record["log_records"]: + if phase["name"] in log_data["logger_name"]: + log_output.append(log_data["message"]) output.extend(["name: " + phase_data["name"], "validators: " + str(phase_data["validators"]), "measured_value: " + str(phase_data["measured_value"]), - "outcome: " + phase_data["outcome"], "\n"]) + "outcome: " + phase_data["outcome"], + "log: " + "\n".join(log_output), "\n"]) if phase["outcome"] == "PASS": test_case = TestCase(phase["name"],