Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

smash stats and ignored message fix #120

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion bin/smash
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@ del standardSetup

global_exitcode_error = False


def main():
import os, os.path, sys
import glob
@@ -181,7 +182,29 @@ def main():

#
###########


class Stats:
def __init__(self):
self.failed = self.passed = self.ignored = self.total = 0

def add(self, returncode, ignored):
if ignored:
self.ignored += 1
elif returncode != 0:
self.failed += 1
else:
self.passed += 1

self.total += 1

def print_stats(self):
#logger.error("IN PRINT")
log_quiet("PASSED: %d" % self.passed)
log_quiet("FAILED: %d" % self.failed)
log_quiet("IGNORED: %d" % self.ignored)

stats = Stats()

if args.dry_run:
log_quiet('*** DRY RUN ***')
smashbox.script.config_log(logging.INFO,hide_password=not args.debug)
@@ -204,6 +227,15 @@ def main():

reporter.testcase_stop(p.returncode)

if config.get('_test_ignored', None):
test_ignored = 1
#logger.error('Not NONE: (%s)'%config.get('_test_ignored', None))
else:
test_ignored = 0
#logger.error('IS NONE')

stats.add(p.returncode, test_ignored)

if p.returncode != 0:
if args.keep_going:
logger.error('Non-zero exit code (%s)'%p.returncode)
@@ -219,6 +251,8 @@ def main():
logFile = os.path.join(config.smashdir, logName)
os.remove(logFile)

config._test_ignored = None


from smashbox.utilities import oc_webdav_url

@@ -317,11 +351,18 @@ def main():
if args.dry_run:
log_quiet('*** DRY RUN ***')

import smashbox.alarming
stats.print_stats()

if stats.failed > stats.failed/20.0 :
smashbox.alarming.send_alarm("smashbox failure","high rate of failed tests (>20%)",logger)

if global_exitcode_error:
sys.exit(global_exitcode_error)

# FIXME: add finally clause with reporter.smashbox_stop()


if __name__ == '__main__':
main()

38 changes: 38 additions & 0 deletions python/smashbox/alarming.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#alarms_sent = set()

def send_alarm(what,msg,logger):

#if what in alarms_sent:
# return

#alarms_sent.add(what)

# add signature
import socket

msg += ""

ALARM_EMAIL = "[email protected]"
#ALARM_EMAIL = "[email protected]"

try:
logger.warning("ALARM: sending alarm to %s: %s %s",ALARM_EMAIL,what,msg)

#if not config.ALARM_EMAIL:
# logging.warning("sending alarms disabled (no ALARM_EMAIL)")
# return

import smtplib
fromaddr='[email protected]'
toaddrs=[ALARM_EMAIL]
email = ("From: %s\r\nTo: %s\r\nSubject: smashbox fail alarm : %s\r\n\r\n" % (fromaddr, ", ".join(toaddrs), what))
email += what + ': ' + msg
server = smtplib.SMTP('localhost')
#server.set_debuglevel(1)
server.sendmail(fromaddr, toaddrs, email)
server.quit()
except Exception,x:
logger.critical("problem sending alarm: %s %s",what,str(x))
alarms_sent.remove(what)


6 changes: 3 additions & 3 deletions python/smashbox/monitoring/kibana_monitoring.py
Original file line number Diff line number Diff line change
@@ -72,15 +72,15 @@ def test_finish(self):
if not self.kibana_monitoring_host:
return

if self._test_ignored is None: # we just want to track tests that we considered fixed
if self._test_ignored: # we just want to track tests that we considered fixed
self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter
else:
if(self.test_results['total_errors']>=1): # A subtest is considered failed with one or more errors
self.test_results['passed'] = 0
self.test_results['failed'] = 1
else:
self.test_results['failed'] = 0
self.test_results['passed'] = 1
else:
self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter

json_results = self.get_json_results()

4 changes: 2 additions & 2 deletions python/smashbox/utilities/__init__.py
Original file line number Diff line number Diff line change
@@ -901,8 +901,8 @@ def expect_does_not_exist(fn):

############ Helper functions to report/document the behaviour of the tests ############

def do_not_report_as_failure(Issue=""):
config._test_ignored = Issue
def do_not_report_as_failure(reason="failure reporting disabled"):
config._test_ignored = reason

############ Smashbox Exceptions ############