diff --git a/bin/smash b/bin/smash index f0eaa0a..f30d400 100755 --- a/bin/smash +++ b/bin/smash @@ -26,6 +26,7 @@ del standardSetup global_exitcode_error = False + def main(): import os, os.path, sys import glob @@ -181,7 +182,29 @@ def main(): # ########### - + + class Stats: + def __init__(self): + self.failed = self.passed = self.ignored = self.total = 0 + + def add(self, returncode, ignored): + if ignored: + self.ignored += 1 + elif returncode != 0: + self.failed += 1 + else: + self.passed += 1 + + self.total += 1 + + def print_stats(self): + #logger.error("IN PRINT") + log_quiet("PASSED: %d" % self.passed) + log_quiet("FAILED: %d" % self.failed) + log_quiet("IGNORED: %d" % self.ignored) + + stats = Stats() + if args.dry_run: log_quiet('*** DRY RUN ***') smashbox.script.config_log(logging.INFO,hide_password=not args.debug) @@ -204,6 +227,15 @@ def main(): reporter.testcase_stop(p.returncode) + if config.get('_test_ignored', None): + test_ignored = 1 + #logger.error('Not NONE: (%s)'%config.get('_test_ignored', None)) + else: + test_ignored = 0 + #logger.error('IS NONE') + + stats.add(p.returncode, test_ignored) + if p.returncode != 0: if args.keep_going: logger.error('Non-zero exit code (%s)'%p.returncode) @@ -219,6 +251,8 @@ def main(): logFile = os.path.join(config.smashdir, logName) os.remove(logFile) + config._test_ignored = None + from smashbox.utilities import oc_webdav_url @@ -317,11 +351,18 @@ def main(): if args.dry_run: log_quiet('*** DRY RUN ***') + import smashbox.alarming + stats.print_stats() + + if stats.failed > stats.failed/20.0 : + smashbox.alarming.send_alarm("smashbox failure","high rate of failed tests (>20%)",logger) + if global_exitcode_error: sys.exit(global_exitcode_error) # FIXME: add finally clause with reporter.smashbox_stop() + if __name__ == '__main__': main() diff --git a/python/smashbox/alarming.py b/python/smashbox/alarming.py new file mode 100644 index 0000000..175fdf0 --- /dev/null +++ b/python/smashbox/alarming.py @@ -0,0 +1,38 @@ +#alarms_sent = set() + +def send_alarm(what,msg,logger): + + #if what in alarms_sent: + # return + + #alarms_sent.add(what) + + # add signature + import socket + + msg += "" + + ALARM_EMAIL = "cernbox-admins@cern.ch" + #ALARM_EMAIL = "georgios.alexandropoulos@cern.ch" + + try: + logger.warning("ALARM: sending alarm to %s: %s %s",ALARM_EMAIL,what,msg) + + #if not config.ALARM_EMAIL: + # logging.warning("sending alarms disabled (no ALARM_EMAIL)") + # return + + import smtplib + fromaddr='cernbox-admins@cern.ch' + toaddrs=[ALARM_EMAIL] + email = ("From: %s\r\nTo: %s\r\nSubject: smashbox fail alarm : %s\r\n\r\n" % (fromaddr, ", ".join(toaddrs), what)) + email += what + ': ' + msg + server = smtplib.SMTP('localhost') + #server.set_debuglevel(1) + server.sendmail(fromaddr, toaddrs, email) + server.quit() + except Exception,x: + logger.critical("problem sending alarm: %s %s",what,str(x)) + alarms_sent.remove(what) + + diff --git a/python/smashbox/monitoring/kibana_monitoring.py b/python/smashbox/monitoring/kibana_monitoring.py index e5a76d0..0a0aa86 100644 --- a/python/smashbox/monitoring/kibana_monitoring.py +++ b/python/smashbox/monitoring/kibana_monitoring.py @@ -72,15 +72,15 @@ def test_finish(self): if not self.kibana_monitoring_host: return - if self._test_ignored is None: # we just want to track tests that we considered fixed + if self._test_ignored: # we just want to track tests that we considered fixed + self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter + else: if(self.test_results['total_errors']>=1): # A subtest is considered failed with one or more errors self.test_results['passed'] = 0 self.test_results['failed'] = 1 else: self.test_results['failed'] = 0 self.test_results['passed'] = 1 - else: - self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter json_results = self.get_json_results() diff --git a/python/smashbox/utilities/__init__.py b/python/smashbox/utilities/__init__.py index 89e5c41..7f15812 100644 --- a/python/smashbox/utilities/__init__.py +++ b/python/smashbox/utilities/__init__.py @@ -901,8 +901,8 @@ def expect_does_not_exist(fn): ############ Helper functions to report/document the behaviour of the tests ############ -def do_not_report_as_failure(Issue=""): - config._test_ignored = Issue +def do_not_report_as_failure(reason="failure reporting disabled"): + config._test_ignored = reason ############ Smashbox Exceptions ############