diff --git a/adr/queries/activedata_usage.query b/adr/queries/activedata_usage.query index 38e7365..4d5d891 100644 --- a/adr/queries/activedata_usage.query +++ b/adr/queries/activedata_usage.query @@ -1,14 +1,32 @@ { - "from":"activedata_requests", - "edges":{ - "value":"timestamp", - "domain":{ - "type":"time", - "min":{$eval: from_date}, - "max":{$eval: to_date}, - "interval":"day" - } - }, - "where":{"gt":{"timestamp":{"date":{$eval: from_date}}}}, - "format":"table" -} + "query": { + "from": "activedata_requests", + "edges": { + "value": "timestamp", + "domain": { + "type": "time", + "min": { + $eval: from + }, + "max": { + $eval: to + }, + "interval": "day" + } + }, + "where": { + "gt": { + "timestamp": { + "date": { + $eval: from + } + } + } + }, + "format": "table" + }, "argument_parser": "\n Usage:\n active_data [--from ] [--to ]\n\n Options:\n\ + \ --from= Starting date to pull data from, defaults to a week ago.\ + \ [default: today-week]\n --to= Ending date to pull data from, defaults\ + \ to now. [default: eod]\n" + +} \ No newline at end of file diff --git a/adr/queries/backout_rate.query b/adr/queries/backout_rate.query index 209a793..0c4fd79 100644 --- a/adr/queries/backout_rate.query +++ b/adr/queries/backout_rate.query @@ -1,24 +1,95 @@ --- -# all push ids -from: repo -select: - - push.id -where: - and: - - in: {branch.name: ["autoland", "mozilla-inbound"]} - - gte: [push.date, {date: {$eval: from_date}}] - - lte: [push.date, {date: {$eval: to_date}}] -limit: 100000 - + { + "query": { + "from": "repo", + "select": [ + "push.id" + ], + "where": { + "and": [ + { + "in": { + "branch.name": [ + "autoland", + "mozilla-inbound" + ] + } + }, + { + "gte": [ + "push.date", + { + "date": { + $eval: from + } + } + ] + }, + { + "lte": [ + "push.date", + { + "date": { + $eval: to + } + } + ] + } + ] + }, + "limit": 100000 + }, + "argument_parser": "\n Usage:\n all_push_ids [--from ] [--to ]\n\n Options:\n\ + \ --from= Starting date to pull data from, defaults to a week ago.\ + \ [default: today-week]\n --to= Ending date to pull data from, defaults\ + \ to now. [default: eod]\n" + } --- -# backout push ids -from: repo -select: - - push.id -where: - and: - - in: {branch.name: ["autoland", "mozilla-inbound"]} - - gte: [push.date, {date: {$eval: from_date}}] - - lte: [push.date, {date: {$eval: to_date}}] - - exists: changeset.backedoutby -limit: 100000 + { + "query": { + "from": "repo", + "select": [ + "push.id" + ], + "where": { + "and": [ + { + "in": { + "branch.name": [ + "autoland", + "mozilla-inbound" + ] + } + }, + { + "gte": [ + "push.date", + { + "date": { + $eval: from + } + } + ] + }, + { + "lte": [ + "push.date", + { + "date": { + $eval: to + } + } + ] + }, + { + "exists": "changeset.backedoutby" + } + ] + }, + "limit": 100000 + }, + "argument_parser": "\n Usage:\n backout_push_ids [--from ] [--to ]\n\n Options:\n\ + \ --from= Starting date to pull data from, defaults to a week ago.\ + \ [default: today-week]\n --to= Ending date to pull data from, defaults\ + \ to now. [default: eod]\n" + } \ No newline at end of file diff --git a/adr/queries/config_durations.query b/adr/queries/config_durations.query index 7492c25..a0856df 100644 --- a/adr/queries/config_durations.query +++ b/adr/queries/config_durations.query @@ -1,15 +1,91 @@ -from: task -select: - - {aggregate: count, name: tasks} - - {aggregate: avg, name: "average minutes", value: {div: {action.duration: 60}}} -groupby: - - "run.machine.platform" - - "build.type" -where: - and: - - in: {repo.branch.name: {$eval: branch}} - - gte: [repo.push.date, {date: {$eval: from_date}}] - - lte: [repo.push.date, {date: {$eval: to_date}}] - - in: {run.suite.name: ["mochitest", "reftest", "web-platform-tests", "web-platform-tests-reftests", "xpcshell", "marionette", "firefox-ui", "cppunittest", "gtest", "jittest", "awsy", "web-platform-tests-wdspec", "test-verification"]} - - in: {build.type: ["opt","debug","pgo","asan"]} -limit: 10000 +{ + "query": { + "from": "task", + "select": [ + { + "aggregate": "count", + "name": "tasks" + }, + { + "aggregate": "avg", + "name": "average minutes", + "value": { + "div": { + "action.duration": 60 + } + } + } + ], + "groupby": [ + "run.machine.platform", + "build.type" + ], + "where": { + "and": [ + { + "in": { + "repo.branch.name": { + $eval: branch + } + } + }, + { + "gte": [ + "repo.push.date", + { + "date": { + $eval: from + } + } + ] + }, + { + "lte": [ + "repo.push.date", + { + "date": { + $eval: to + } + } + ] + }, + { + "in": { + "run.suite.name": [ + "mochitest", + "reftest", + "web-platform-tests", + "web-platform-tests-reftests", + "xpcshell", + "marionette", + "firefox-ui", + "cppunittest", + "gtest", + "jittest", + "awsy", + "web-platform-tests-wdspec", + "test-verification" + ] + } + }, + { + "in": { + "build.type": [ + "opt", + "debug", + "pgo", + "asan" + ] + } + } + ] + }, + "limit": 10000 + }, + "argument_parser": "\n Argument parser for the config_durations query.\n \n Usage:\n config_durations\ + \ [--from FROM_DATE] [--to TO_DATE] [--branch BRANCH...]\n \n Options:\n \ + \ --from=FROM_DATE Starting date to pull data from, defaults to a week ago. [default:\ + \ today-week]\n --to=TO_DATE Ending date to pull data from, defaults to now.\ +\ [default: eod]\n --branch=BRANCH Branches to query results from. [default:\ +\ mozilla-central]\n " +} \ No newline at end of file diff --git a/adr/query.py b/adr/query.py index fad30cd..103689a 100644 --- a/adr/query.py +++ b/adr/query.py @@ -9,7 +9,7 @@ import jsone import requests import yaml - +from adr.util.modified_docopt import docopt from adr.formatter import all_formatters from adr.errors import MissingDataError @@ -76,11 +76,14 @@ def load_query(name): :yields dict query: dictionary representation of yaml query. """ with open(os.path.join(QUERY_DIR, name + '.query')) as fh: - for query in yaml.load_all(fh): - yield query + for record in yaml.load_all(fh): + try: + yield record["argument_parser"], record["query"] + except KeyError: + yield '', record -def run_query(name, config, **context): +def run_query(name, config, *args, **kwargs): """Loads and runs the specified query, yielding the result. Given name of a query, this method will first read the query @@ -97,9 +100,14 @@ def run_query(name, config, **context): :param dict context: dictionary of ActiveData configs. :yields str: json-formatted string. """ - for query in load_query(name): + for argument_parser, query in load_query(name): # If limit is in the context, override the queries' value. We do this # to keep the results down to a sane level when testing queries. + if len(argument_parser) != 0: + parsed_arguments = docopt(argument_parser, argv=args) + context = {k.replace('--', ''): v for k, v in parsed_arguments.items()} + else: + context = kwargs if 'limit' in context: query['limit'] = context['limit'] if 'format' in context: diff --git a/adr/recipes/activedata_usage.py b/adr/recipes/activedata_usage.py index 76595b5..fe88f91 100644 --- a/adr/recipes/activedata_usage.py +++ b/adr/recipes/activedata_usage.py @@ -1,21 +1,15 @@ """ -Show ActiveData query usage, by day + Show ActiveData query usage, by day -.. code-block:: bash + .. code-block:: bash adr activedata_usage [--from [--to ]] -""" + """ from __future__ import print_function, absolute_import - -from ..recipe import RecipeParser from ..query import run_query def run(args, config): - parser = RecipeParser('date') - args = parser.parse_args(args) - - query_args = vars(args) - response = next(run_query('activedata_usage', config, **query_args)) + response = next(run_query('activedata_usage', config, *args)) return [response['header']] + response['data'] diff --git a/adr/recipes/backout_rate.py b/adr/recipes/backout_rate.py index 963d880..5b329c6 100644 --- a/adr/recipes/backout_rate.py +++ b/adr/recipes/backout_rate.py @@ -9,18 +9,12 @@ `View Results `__ """ from __future__ import print_function, absolute_import - -from ..recipe import RecipeParser from ..query import run_query def run(args, config): - parser = RecipeParser('date') - args = parser.parse_args(args) - - query_args = vars(args) - query = run_query('backout_rate', config, **query_args) + query = run_query('backout_rate', config, *args) pushes = len(set(next(query)['data']['push.id'])) backouts = len(set(next(query)['data']['push.id'])) backout_rate = round((float(backouts) / pushes) * 100, 2) diff --git a/adr/recipes/config_durations.py b/adr/recipes/config_durations.py index b7252d1..0c36342 100644 --- a/adr/recipes/config_durations.py +++ b/adr/recipes/config_durations.py @@ -6,23 +6,29 @@ adr config_durations [--branch ] """ from __future__ import print_function, absolute_import - -from ..recipe import RecipeParser +from adr.util.modified_docopt import docopt from ..query import run_query +argument_parser = """ + Argument parser for the config_durations recipe. + + Usage: + config_durations [--limit LIMIT] [--sort-key SORTKEY] + + Options: + --limit=LIMIT Maximum number of jobs to return. [default: 50] + --sort-key=SORTKEY Key to sort on (int, 0-based index). [default: 4] + """ + + def run(args, config): - parser = RecipeParser('date', 'branch') - parser.add_argument('--limit', type=int, default=50, - help="Maximum number of jobs to return") - parser.add_argument('--sort-key', type=int, default=4, - help="Key to sort on (int, 0-based index)") - args = parser.parse_args(args) - query_args = vars(args) - limit = query_args.pop('limit') + limit = int(docopt(argument_parser, args)['--limit']) + sort_key = int(docopt(argument_parser, args)['--sort-key']) + + data = next(run_query('config_durations', config, *args))['data'] - data = next(run_query('config_durations', config, **query_args))['data'] result = [] for record in data: if isinstance(record[1], list): @@ -35,6 +41,6 @@ def run(args, config): record.append(int(round(record[2] * record[3], 0))) result.append(record) - result = sorted(result, key=lambda k: k[args.sort_key], reverse=True)[:limit] + result = sorted(result, key=lambda k: k[sort_key], reverse=True)[:limit] result.insert(0, ['Platform', 'Type', 'Num Jobs', 'Average Hours', 'Total Hours']) return result diff --git a/adr/util/modified_docopt.py b/adr/util/modified_docopt.py new file mode 100644 index 0000000..df8f68e --- /dev/null +++ b/adr/util/modified_docopt.py @@ -0,0 +1,21 @@ +from docopt import DocoptExit, Option, AnyOptions, TokenStream, Dict +from docopt import parse_defaults, parse_pattern, parse_argv, extras +from docopt import formal_usage, printable_usage, sys + + +def docopt(doc, argv=None, help=True, version=None, options_first=False): + if argv is None: + argv = sys.argv[1:] + DocoptExit.usage = printable_usage(doc) + options = parse_defaults(doc) + pattern = parse_pattern(formal_usage(DocoptExit.usage), options) + argv = parse_argv(TokenStream(argv, DocoptExit), list(options), options_first) + pattern_options = set(pattern.flat(Option)) + for ao in pattern.flat(AnyOptions): + doc_options = parse_defaults(doc) + ao.children = list(set(doc_options) - pattern_options) + extras(help, version, argv, doc) + matched, left, collected = pattern.fix().match(argv) + if matched: + return Dict((a.name, a.value) for a in (pattern.flat() + collected)) + DocoptExit() diff --git a/setup.py b/setup.py index 72badfd..1c368f2 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ 'json-e >= 2.3.2', 'requests >= 2.18.3', 'terminaltables >= 3.1.0', + 'docopt==0.6.2', 'pyyaml', 'beautifulsoup4', 'flask',