wktesthunter

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  Copyright 2014 Igalia S.L.
#  Carlos Alberto Lopez Perez <clopez@igalia.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# Run with python3 if available (is faster)
import sys
import os
try:
    if sys.version_info[0] < 3:
        os.execvp('python3', ['python3', __file__] + sys.argv[1:])
except:
    # No python3 :(
    pass

import json
import argparse
from collections import OrderedDict

extrahelp = '''
About the colors:

    - A red color means that the test result and the expectation didn't matched.
    - A green color means that the test result was expected.
      - If 'FAIL' is on the expectation, then the test matches the expectation
        if the result is any of 'FAIL', 'TEXT', 'IMAGE+TEXT' or 'AUDIO'.
    - No color means UNKNOWN (no data)

    * To understand this better, run with --showexpected to check what was
      the expectation for that test at the moment.


The meaning of the possible results for a test on a revision is:

    "UNKNOWN" means that we have no data for that revision. This means that
    the bot didn't ran the test for this revision. This could have been
    caused by a compilation failure on that revision, or could have been
    that the bot skipped that revision because of the size of the build queue
    at that moment, or maybe the bot was offline.... etc.


    "NOERROR" means that we don't know the exactly result of this test on that revision,
    but we know that the complete set of layout tests finished on that revision and no
    specific problem was reported with this test.
    This usually means one of this four options:
    - The test executed without failure (if the test was not flagged on TestExpectations).
    - The test gave the failure expected on TestExpectations.
    - The test didn't exist by that revision.
    - The test was marked to be skipped.


    "IMAGE" means ImageOnlyFailure

    "MISSING" means Missing results.

    "PASS" means the test passed. This can be bad (if the color is red means that the test
     was not expected to pass according to TestExpectations).
     Run with --showexpected to check what was the expectation for that test at the moment.

    "TEXT" means text failure.

    "CRASH" means the test crashed.

    "AUDIO" means audio failure

    "FAIL" means failure

Note for lazy people:

    * As an extra goodie from python's argparse, the parameters can be shorted to any length
      meanwhile there is no possible confusion between them. For example, you can use any of
      "--showexpected", "--show" or "--s"; or you can use "--mergeunknown", "--merge" or "--m"
'''


def iprint(firstrev,lastrev,result,startcolor):

    if lastrev == None:
        startstr=("r%d" %firstrev)
    else:
        startstr=("[r%d-r%d]" %(firstrev,lastrev))

    if startcolor != None:
        endcolor='\033[0m'
    else:
        endcolor=''
        startcolor=''

    print('%s%s%s%s%s' %(startcolor,startstr," "*(30-len(startstr)),result,endcolor))




def main(test, debug, detail, mergeunknown, showexpected, usecolor):
    resultsfortests = {}
    resultsfortests[test]=OrderedDict()
    testparts=test.split('/')
    if len(testparts) < 2 or "." not in testparts[len(testparts)-1]:
        raise ValueError("Test should be in the format path/test.html. Only one test accepted as parameter (no wildcards or directories)")
    bot="GTK_Linux_64-bit_Release"
    jsonresults=os.listdir('jsonresults')
    jsonresults.sort()

    if mergeunknown:
        noerror="NOERROR/UNKNOWN"
        unknown=noerror
    else:
        noerror="NOERROR"
        unknown="UNKNOWN"

    for jsonresult in jsonresults:
        if jsonresult.startswith(bot+"_full_results_") and jsonresult.endswith('.json'):
            (revision, buildnumber) = jsonresult.split(bot+"_full_results_")[1].split(".json")[0].split('_')
            revision=int(revision.strip('r'))
            buildnumber=int(buildnumber.strip('b'))
            # Read file
            json_file=open(os.path.join('jsonresults', jsonresult))
            json_data=json_file.read()
            json_file.close()
            # Clean it
            json_data=json_data.split('ADD_RESULTS(')[1]
            json_data=json_data.split(');')[0]
            # Parse it
            json_parsed = json.loads(json_data)
            # Sanity check
            if int(json_parsed['revision']) != revision:
                if debug: print ("WARNING: Parsed revision %s don't match expected one %d for file %s" %(json_parsed['revision'],revision,jsonresult))
                revision=int(json_parsed['revision'])

            if revision in resultsfortests[test]:
                if debug: print ("WARNING: Data for revision %d duplicated. Picking last\n" %revision)
            else:
                resultsfortests[test][revision]={}

            keytest=json_parsed['tests']
            try:
                for testpart in testparts:
                    keytest=keytest[testpart]
            except KeyError:
                # This test didn't failed (or wasn't ran) on this rev.
                resultsfortests[test][revision]['result']=noerror
                continue

            resultsfortests[test][revision]['result']=keytest['actual']
            if 'expected' in keytest:
                resultsfortests[test][revision]['expected']=keytest['expected']


    print ("Results for test: %s\n" %test)
    minrev=list(resultsfortests[test].keys())[0]
    maxrev=list(resultsfortests[test].keys())[len(resultsfortests[test])-1]
    if debug: print ("minrev es %d y maxrev %d" %(minrev,maxrev))
    lastprinted=unknown
    lastrevprinted=0
    # trick to end the print the last interval on the loop.
    resultsfortests[test][maxrev+1]={}
    resultsfortests[test][maxrev+1]['result']='END'
    for revision in range(minrev,maxrev+2):
        if revision in resultsfortests[test]:
            toprint=resultsfortests[test][revision]['result']
            if showexpected and 'expected' in resultsfortests[test][revision]:
                toprint = "%s (Expected: %s)" %(toprint,resultsfortests[test][revision]['expected'])
        else:
                toprint=unknown
        if lastprinted==toprint:
            continue
        else:
            if detail or (lastprinted != unknown and lastprinted != noerror):
                color=None
                if usecolor:
                    # We are going to print. Choose the color
                    if lastprinted != unknown:
                        color='\033[0;32m' # green
                    try:
                        if 'expected' in resultsfortests[test][revision-1]:
                            for value in resultsfortests[test][revision-1]['result'].split(" "):
                                if ( (value not in resultsfortests[test][revision-1]['expected']) and # we check also for "[TEXT|IMAGE+TEXT|AUDIO]==FAIL"
                                     ( (value not in ('TEXT', 'IMAGE+TEXT', 'AUDIO')) or ('FAIL' not in resultsfortests[test][revision-1]['expected']) ) ):
                                    color='\033[0;31m' # red
                                    if debug: print ("INFO: Marking test % as green on rev r%d, because actual value \"%s\" is in expected \"%s\" "
                                        %(test,revision-1,resultsfortests[test][revision-1]['result'],resultsfortests[test][revision-1]['expected']))
                    # This means we are going to print an unknown interval
                    except KeyError:
                        pass

                if lastprinted != unknown or not ignoreunknown:
                    if revision-lastrevprinted > 2:
                        iprint (lastrevprinted+1,revision-1,lastprinted,color)
                    else:
                        iprint (revision-1,None,lastprinted,color)
            lastrevprinted=revision-1
            lastprinted=toprint
    print("\n")
    return 0

if __name__ == '__main__':

    parser = argparse.ArgumentParser(epilog=extrahelp, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("--debug", help="Print debug messages.", action="store_true")
    parser.add_argument("--onlyprinterr", help="Don't print NOERROR or UNKNOWN results.", action="store_true")
    parser.add_argument("--mergeunknown", help="If set, will join the UNKNOWN results (no data for this revision)\n"
                                               "with the NOERROR results (we have data, but no error/result was reported).\n", action="store_true")
    parser.add_argument("--ignoreunknown", help="If set, will not print UNKNOWN (no data for this revision)\n"
                                          "This option disregards --mergeunknown.\n", action="store_true")
    parser.add_argument("--showexpected", help="Print not only the test results, but also the expected result.", action="store_true")
    parser.add_argument("--nocolor", help="Don't print colors", action="store_true")
    parser.add_argument("test_name", type=str, help="Name for the test (as specified on run-webkit-tests).")
    args = parser.parse_args()

    if args.debug: debug = True
    else: debug = False
    if args.onlyprinterr: detail = False
    else: detail = True
    if args.mergeunknown: mergeunknown = True
    else: mergeunknown = False
    if args.ignoreunknown: ignoreunknown = True;
    else: ignoreunknown = False
    if args.showexpected: showexpected = True
    else: showexpected = False
    if args.nocolor: usecolor = False
    else: usecolor = True

    main(args.test_name, debug, detail, mergeunknown, showexpected, usecolor)