Merge pull request #16 from crccheck/lazier-plumbum

plumbum cleanup

Makefile

@@ -9,7 +9,7 @@ clean:
 
 
 test:
-	python test_leadbutt.py
+	python -m unittest discover
 
 
 # Release Instructions:
@@ -28,5 +28,5 @@ build:
 
 # makes it easier to test setup.py's entry points
 reinstall:
-	pip uninstall cloudwatch-to-graphite --yes
+	-pip uninstall cloudwatch-to-graphite --yes
 	pip install .

config.yaml.j2.example → ec2_sample.yml.j2

@@ -1,7 +1,7 @@
-{# This is a Jinja2 comment
+{#- This is a Jinja2 comment
   To use this template, you'd run something like:
-  python plumbum.py config.yaml.j2.example ec2 us-west-2
- #}
+  plumbum ec2_sample.yml.j2 ec2 us-west-2
+#}
 # Sample config.yaml
 #
 Auth:
@@ -17,7 +17,7 @@ Metrics:
   Dimensions:
     InstanceId: "{{ instance.id }}"
   Options:
-    {# I'm assuming my tag names are safe to use as metric names here #}
+    {#- I'm assuming my tag names are safe to use as metric names here #}
     Formatter: 'cloudwatch.%(Namespace)s.{{ instance.tags['Name'] }}.%(MetricName)s.%(statistic)s.%(Unit)s'
     Period: 5
-{%- endfor %}
+{% endfor %}

elb_sample.yml.j2

@@ -0,0 +1,19 @@
+Metrics:
+{%- for elb in resources %}
+- Namespace: "AWS/ELB"
+  MetricName: "RequestCount"
+  Statistics: "Sum"
+  Unit: "Count"
+  Dimensions:
+    LoadBalancerName: "{{ elb.name }}"
+- Namespace: "AWS/ELB"
+  MetricName: "HTTPCode_Backend_5XX"
+  Statistics: "Sum"
+  Unit: "Count"
+  Dimensions:
+    LoadBalancerName: "{{ elb.name }}"
+  Options:
+    Period: 5
+{% endfor %}
+Auth:
+  region: "{{ region }}""

plumbum.py

@@ -1,7 +1,13 @@
 # -*- coding: UTF-8 -*-
 """
 Usage:
-  plumbum <template> <namespace> [options]...
+  plumbum <template> <namespace> [region] [options]...
+
+Options:
+  template   path to the jinja2 template
+  namespace  AWS namespace. Currently supports: elb, ec2, rds
+  region     AWS region [default: us-east-1]
+  options    key value combinations, they can be tags or any other property
 
 Examples:
 
@@ -12,9 +18,18 @@
 
 Outputs to stdout.
 
+About Templates:
+
+Templates are used to generate config.yml files based on running resources.
+They're written in jinja2, and have these variables available:
+
+  filters    A dictionary of the filters that were passed in
+  region     The region the resource is located in
+  resources  A list of the resources as boto objects
 """
 from __future__ import unicode_literals
 
+import re
 import sys
 
 import boto
@@ -24,6 +39,10 @@
 import jinja2
 
 
+# DEFAULT_NAMESPACE = 'ec2'  # TODO
+DEFAULT_REGION = 'us-east-1'
+
+
 def get_property_func(key):
     """
     Get the accessor function for an instance to look for `key`.
@@ -47,81 +66,101 @@ def filter_instance(instance):
 
 
 def lookup(instances, filter_by=None):
-    if filter_by:
+    if filter_by is not None:
         return filter(filter_key(filter_by), instances)
     return instances
 
 
-def get_options(input_args):
-    filter_by_kwargs = {}
-    for arg in input_args:
+def interpret_options(options):
+    """Parse all the command line options."""
+    # template always has to be index 0
+    template = options[0]
+    # namespace always has to be index 1. Support 'ec2' (human friendly) and
+    # 'AWS/EC2' (how CloudWatch natively calls these things)
+    namespace = options[1].rsplit('/', 2)[-1].lower()
+    next_idx = 2
+    # region might be index 2
+    region = ''
+    if len(options) > 2 and re.match(r'^\w+\-[\w\-]+\-\d+$', options[2]):
+        region = options[2]
+        next_idx += 1
+    else:
+        next_idx = 2
+    region = region or boto.config.get('Boto', 'ec2_region_name', 'us-east-1')
+
+    filter_by = {}
+    extras = []
+    for arg in options[next_idx:]:
         if arg.startswith('-'):
-            # ignore options
-            continue
-        if '=' in arg:
+            # throw these away for now
+            extras.append(arg)
+        elif '=' in arg:
             key, value = arg.split('=', 2)
-            filter_by_kwargs[key] = value
-    return filter_by_kwargs
+            filter_by[key] = value
+        else:
+            # throw these away for now
+            extras.append(arg)
+
+    return template, namespace, region, filter_by, extras
 
 
 def list_ec2(region, filter_by_kwargs):
+    """List running ec2 instances."""
     conn = boto.ec2.connect_to_region(region)
     instances = conn.get_only_instances()
     return lookup(instances, filter_by=filter_by_kwargs)
 
 
 def list_elb(region, filter_by_kwargs):
+    """List all load balancers."""
     conn = boto.ec2.elb.connect_to_region(region)
     instances = conn.get_all_load_balancers()
     return lookup(instances, filter_by=filter_by_kwargs)
 
 
 def list_rds(region, filter_by_kwargs):
+    """List all RDS thingys."""
     conn = boto.rds.connect_to_region(region)
     instances = conn.get_all_dbinstances()
     return lookup(instances, filter_by=filter_by_kwargs)
 
 
+list_resources = {
+    'ec2': list_ec2,
+    'elb': list_elb,
+    'rds': list_rds,
+}
+
+
 def main():
     if len(sys.argv) < 3:
-        print __doc__
+        print(__doc__)
         sys.exit()
-    options = sys.argv[1:]
-    template = options[0]
-    namespace = options[1].lower()
-    region = ''
-    if '=' in options[2]:
-        filters = get_options(options[2:])
-    else:
-        region = options[2]
-        filters = get_options(options[3:])
+
+    template, namespace, region, filters, __ = interpret_options(sys.argv[1:])
 
     # get the template first so this can fail before making a network request
     loader = jinja2.FileSystemLoader('.')
     jinja2_env = jinja2.Environment(loader=loader)
     template = jinja2_env.get_template(template)
 
     # insure a valid region is set
-    region = region or boto.config.get('Boto', 'ec2_region_name', 'us-east-1')
     if not region in [r.name for r in boto.ec2.regions()]:
         raise ValueError("Invalid region:{0}".format(region))
 
     # should I be using ARNs?
-    if namespace in ('ec2', 'aws/ec2'):
-        resources = list_ec2(region, filters)
-    elif namespace in ('elb', 'aws/elb'):
-        resources = list_elb(region, filters)
-    elif namespace in ('rds', 'aws/rds'):
-        resources = list_rds(region, filters)
-    else:
-        # TODO
+    try:
+        resources = list_resources[namespace](region, filters)
+    except KeyError:
+        print('ERROR: AWS namespace "{}" not supported or does not exist'
+            .format(namespace))
         sys.exit(1)
 
-    print template.render({
+    print(template.render({
         'filters': filters,
+        'region': region,       # Use for Auth config section if needed
         'resources': resources,
-        'region': region,       #Use for Auth config section if needed
-    })
+    }))
 
 
 if __name__ == '__main__':

test_plumbum.py

@@ -0,0 +1,78 @@
+# -*- coding: UTF-8 -*-
+"""
+Tests for Cloudwatch to Graphite (leadbutt)
+
+WISHLIST: supress chatty stderr and stdout in tests
+"""
+from __future__ import unicode_literals
+
+import unittest
+
+import plumbum
+
+
+class get_cli_optionsTest(unittest.TestCase):
+    def test_trivial_case(self):
+        argv = []
+        with self.assertRaises(IndexError):
+            plumbum.interpret_options(argv)
+
+    def test_minimal(self):
+        argv = ['foo.j2', 'ec2']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(templ, 'foo.j2')
+        self.assertEqual(ns, 'ec2')
+        self.assertEqual(region, plumbum.DEFAULT_REGION)
+        self.assertFalse(len(filter_by))
+        self.assertFalse(len(extras))
+
+    def test_namespace_can_use_cloudwatch_syntax(self):
+        argv = ['foo.j2', 'AWS/EC2']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(templ, 'foo.j2')
+        self.assertEqual(ns, 'ec2')
+
+    def test_region_can_be_specified(self):
+        argv = ['foo.j2', 'ec2', 'avengers-west-2']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(templ, 'foo.j2')
+        self.assertEqual(ns, 'ec2')
+        self.assertEqual(region, 'avengers-west-2')
+        self.assertFalse(len(filter_by))
+        self.assertFalse(len(extras))
+
+        # more regions
+        argv = ['foo.j2', 'ec2', 'us-east-1']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(region, 'us-east-1')
+        argv = ['foo.j2', 'ec2', 'cn-north-1']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(region, 'cn-north-1')
+        argv = ['foo.j2', 'ec2', 'ap-northeast-1']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(region, 'ap-northeast-1')
+        argv = ['foo.j2', 'ec2', 'us-gov-west-1']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(region, 'us-gov-west-1')
+
+    def test_filters_and_extras_found(self):
+        argv = ['foo.j2', 'ec2', 'bar=mars', '--whee', 'xyzzy']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(templ, 'foo.j2')
+        self.assertEqual(ns, 'ec2')
+        self.assertEqual(region, plumbum.DEFAULT_REGION)
+        self.assertEqual(len(filter_by), 1)
+        self.assertEqual(len(extras), 2)
+
+    def test_filters_and_extras_with_region_specified(self):
+        argv = ['foo.j2', 'ec2', 'avengers-west-2', 'bar=mars', '--whee', 'xyzzy']
+        templ, ns, region, filter_by, extras = plumbum.interpret_options(argv)
+        self.assertEqual(templ, 'foo.j2')
+        self.assertEqual(ns, 'ec2')
+        self.assertEqual(region, 'avengers-west-2')
+        self.assertEqual(len(filter_by), 1)
+        self.assertEqual(len(extras), 2)
+
+
+if __name__ == '__main__':
+    unittest.main()

tox.ini

@@ -2,7 +2,9 @@
 envlist = py27, py34
 
 [testenv]
-commands = {envpython} test_leadbutt.py
-deps = mock
+commands = {envpython} -m unittest discover
+deps =
+    mock
+    jinja2
 setenv =
     TOX_TEST_ENTRYPOINT = 1