Issue40

adr/queries/activedata_usage.query

@@ -1,14 +1,32 @@
 {
-  "from":"activedata_requests",
-  "edges":{
-    "value":"timestamp",
-    "domain":{
-      "type":"time",
-      "min":{$eval: from_date},
-      "max":{$eval: to_date},
-      "interval":"day"
-    }
-  },
-  "where":{"gt":{"timestamp":{"date":{$eval: from_date}}}},
-  "format":"table"
-}
+  "query": {
+    "from": "activedata_requests",
+    "edges": {
+      "value": "timestamp",
+      "domain": {
+        "type": "time",
+        "min": {
+          $eval: from
+        },
+        "max": {
+          $eval: to
+        },
+        "interval": "day"
+      }
+    },
+    "where": {
+      "gt": {
+        "timestamp": {
+          "date": {
+            $eval: from
+          }
+        }
+      }
+    },
+    "format": "table"
+  }, "argument_parser": "\n    Usage:\n      active_data [--from <from_date>] [--to <to_date>]\n\n    Options:\n\
+  \      --from=<from_date>  Starting date to pull data from, defaults to a week ago.\
+  \ [default: today-week]\n      --to=<to_date>  Ending date to pull data from, defaults\
+  \ to now. [default: eod]\n"
+
+}

adr/queries/backout_rate.query

@@ -1,24 +1,95 @@
 ---
-# all push ids
-from: repo
-select:
-    - push.id
-where:
-    and:
-        - in: {branch.name: ["autoland", "mozilla-inbound"]}
-        - gte: [push.date, {date: {$eval: from_date}}]
-        - lte: [push.date, {date: {$eval: to_date}}]
-limit: 100000
-
+  {
+    "query": {
+      "from": "repo",
+      "select": [
+        "push.id"
+      ],
+      "where": {
+        "and": [
+          {
+            "in": {
+              "branch.name": [
+                "autoland",
+                "mozilla-inbound"
+              ]
+            }
+          },
+          {
+            "gte": [
+              "push.date",
+              {
+                "date": {
+                  $eval: from
+                }
+              }
+            ]
+          },
+          {
+            "lte": [
+              "push.date",
+              {
+                "date": {
+                  $eval: to
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "limit": 100000
+    },
+    "argument_parser": "\n    Usage:\n      all_push_ids [--from <from_date>] [--to <to_date>]\n\n    Options:\n\
+    \      --from=<from_date>  Starting date to pull data from, defaults to a week ago.\
+    \ [default: today-week]\n      --to=<to_date>  Ending date to pull data from, defaults\
+    \ to now. [default: eod]\n"
+  }
 ---
-# backout push ids
-from: repo
-select:
-    - push.id
-where:
-    and:
-        - in: {branch.name: ["autoland", "mozilla-inbound"]}
-        - gte: [push.date, {date: {$eval: from_date}}]
-        - lte: [push.date, {date: {$eval: to_date}}]
-        - exists: changeset.backedoutby
-limit: 100000
+  {
+    "query": {
+      "from": "repo",
+      "select": [
+        "push.id"
+      ],
+      "where": {
+        "and": [
+          {
+            "in": {
+              "branch.name": [
+                "autoland",
+                "mozilla-inbound"
+              ]
+            }
+          },
+          {
+            "gte": [
+              "push.date",
+              {
+                "date": {
+                  $eval: from
+                }
+              }
+            ]
+          },
+          {
+            "lte": [
+              "push.date",
+              {
+                "date": {
+                  $eval: to
+                }
+              }
+            ]
+          },
+          {
+            "exists": "changeset.backedoutby"
+          }
+        ]
+      },
+      "limit": 100000
+    },
+    "argument_parser": "\n    Usage:\n      backout_push_ids [--from <from_date>] [--to <to_date>]\n\n    Options:\n\
+    \      --from=<from_date>  Starting date to pull data from, defaults to a week ago.\
+    \ [default: today-week]\n      --to=<to_date>  Ending date to pull data from, defaults\
+    \ to now. [default: eod]\n"
+  }

adr/queries/config_durations.query

@@ -1,15 +1,91 @@
-from: task
-select:
-    - {aggregate: count, name: tasks}
-    - {aggregate: avg, name: "average minutes", value: {div: {action.duration: 60}}}
-groupby:
-    - "run.machine.platform"
-    - "build.type"
-where:
-    and:
-        - in: {repo.branch.name: {$eval: branch}}
-        - gte: [repo.push.date, {date: {$eval: from_date}}]
-        - lte: [repo.push.date, {date: {$eval: to_date}}]
-        - in: {run.suite.name: ["mochitest", "reftest", "web-platform-tests", "web-platform-tests-reftests", "xpcshell", "marionette", "firefox-ui", "cppunittest", "gtest", "jittest", "awsy", "web-platform-tests-wdspec", "test-verification"]}
-        - in: {build.type: ["opt","debug","pgo","asan"]}
-limit: 10000
+{
+  "query": {
+    "from": "task",
+    "select": [
+      {
+        "aggregate": "count",
+        "name": "tasks"
+      },
+      {
+        "aggregate": "avg",
+        "name": "average minutes",
+        "value": {
+          "div": {
+            "action.duration": 60
+          }
+        }
+      }
+    ],
+    "groupby": [
+      "run.machine.platform",
+      "build.type"
+    ],
+    "where": {
+      "and": [
+        {
+          "in": {
+            "repo.branch.name": {
+              $eval: branch
+            }
+          }
+        },
+        {
+          "gte": [
+            "repo.push.date",
+            {
+              "date": {
+                $eval: from
+              }
+            }
+          ]
+        },
+        {
+          "lte": [
+            "repo.push.date",
+            {
+              "date": {
+                $eval: to
+              }
+            }
+          ]
+        },
+        {
+          "in": {
+            "run.suite.name": [
+              "mochitest",
+              "reftest",
+              "web-platform-tests",
+              "web-platform-tests-reftests",
+              "xpcshell",
+              "marionette",
+              "firefox-ui",
+              "cppunittest",
+              "gtest",
+              "jittest",
+              "awsy",
+              "web-platform-tests-wdspec",
+              "test-verification"
+            ]
+          }
+        },
+        {
+          "in": {
+            "build.type": [
+              "opt",
+              "debug",
+              "pgo",
+              "asan"
+            ]
+          }
+        }
+      ]
+    },
+    "limit": 10000
+  },
+  "argument_parser": "\n    Argument parser for the config_durations query.\n    \n    Usage:\n    config_durations\
+  \ [--from FROM_DATE] [--to TO_DATE] [--branch BRANCH...]\n    \n    Options:\n \
+  \   --from=FROM_DATE  Starting date to pull data from, defaults to a week ago. [default:\
+  \ today-week]\n    --to=TO_DATE  Ending date to pull data from, defaults to now.\
+\ [default: eod]\n    --branch=BRANCH  Branches to query results from. [default:\
+\ mozilla-central]\n    "
+}

adr/query.py

@@ -9,7 +9,7 @@
 import jsone
 import requests
 import yaml
-
+from adr.util.modified_docopt import docopt
 from adr.formatter import all_formatters
 from adr.errors import MissingDataError
 
@@ -76,11 +76,14 @@ def load_query(name):
     :yields dict query: dictionary representation of yaml query.
     """
     with open(os.path.join(QUERY_DIR, name + '.query')) as fh:
-        for query in yaml.load_all(fh):
-            yield query
+        for record in yaml.load_all(fh):
+            try:
+                yield record["argument_parser"], record["query"]
+            except KeyError:
+                yield '', record
 
 
-def run_query(name, config, **context):
+def run_query(name, config, *args, **kwargs):
     """Loads and runs the specified query, yielding the result.
 
     Given name of a query, this method will first read the query
@@ -97,9 +100,14 @@ def run_query(name, config, **context):
     :param dict context: dictionary of ActiveData configs.
     :yields str: json-formatted string.
     """
-    for query in load_query(name):
+    for argument_parser, query in load_query(name):
         # If limit is in the context, override the queries' value. We do this
         # to keep the results down to a sane level when testing queries.
+        if len(argument_parser) != 0:
+            parsed_arguments = docopt(argument_parser, argv=args)
+            context = {k.replace('--', ''): v for k, v in parsed_arguments.items()}
+        else:
+            context = kwargs
         if 'limit' in context:
             query['limit'] = context['limit']
         if 'format' in context:

adr/recipes/activedata_usage.py

@@ -1,21 +1,15 @@
 """
-Show ActiveData query usage, by day
+    Show ActiveData query usage, by day
 
-.. code-block:: bash
+    .. code-block:: bash
 
     adr activedata_usage [--from <date> [--to <date>]]
-"""
+    """
 
 from __future__ import print_function, absolute_import
-
-from ..recipe import RecipeParser
 from ..query import run_query
 
 
 def run(args, config):
-    parser = RecipeParser('date')
-    args = parser.parse_args(args)
-
-    query_args = vars(args)
-    response = next(run_query('activedata_usage', config, **query_args))
+    response = next(run_query('activedata_usage', config, *args))
     return [response['header']] + response['data']

adr/recipes/backout_rate.py

@@ -9,18 +9,12 @@
 `View Results <https://mozilla.github.io/active-data-recipes/#backout-rate>`__
 """
 from __future__ import print_function, absolute_import
-
-from ..recipe import RecipeParser
 from ..query import run_query
 
 
 def run(args, config):
-    parser = RecipeParser('date')
-    args = parser.parse_args(args)
-
-    query_args = vars(args)
-    query = run_query('backout_rate', config, **query_args)
 
+    query = run_query('backout_rate', config, *args)
     pushes = len(set(next(query)['data']['push.id']))
     backouts = len(set(next(query)['data']['push.id']))
     backout_rate = round((float(backouts) / pushes) * 100, 2)