From 949de3711779f2c1306b3c1186f92bd5ee345361 Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Thu, 18 Feb 2016 15:30:37 +0100 Subject: [PATCH 1/9] README: update MI -> Mantl --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 09f985e..c02a897 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ Or in a playbook: ## Releases `terraform.py` is with matching versions of -[microservices-infrastructure](https://github.com/CiscoCloud/microservices-infrastructure). -Run `terraform.py --version` to report the current version. +[mantl](https://github.com/CiscoCloud/mantl). Run `terraform.py --version` to +report the current version. ## Adding a new provider @@ -72,7 +72,7 @@ calls them.) A parser is a function that takes a resource (a dictionary processed from JSON) and outputs a tuple of `(name, attributes, groups)`. The parser should be decorated with the `parses` decorator, which takes the name of the resource (EG `aws_instance`). It should also be decorated with -`calculate_mi_vars`. +`calculate_mantl_vars`. As a guideline, `terraform.py` should require no resources outside the Python standard distribution so it can just be copied in wherever it's needed. @@ -120,13 +120,13 @@ a list. Given keys like this and the prefix "keys": ... "tags.#": "2", - "tags.2783239913": "mi", + "tags.2783239913": "mantl", "tags.3990563915": "control", ... `parse_list` would return this: - ["mi", "control"] + ["mantl", "control"] ## License From 3f3d69f1c01d88cec45dbb842013220f475a68b3 Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Thu, 18 Feb 2016 15:30:55 +0100 Subject: [PATCH 2/9] terraform.py: MI -> Mantl --- terraform.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/terraform.py b/terraform.py index 3244ce9..efd4db4 100755 --- a/terraform.py +++ b/terraform.py @@ -75,7 +75,7 @@ def inner(func): return inner -def calculate_mi_vars(func): +def calculate_mantl_vars(func): """calculate Mantl vars""" @wraps(func) @@ -139,7 +139,7 @@ def parse_bool(string_form): @parses('digitalocean_droplet') -@calculate_mi_vars +@calculate_mantl_vars def digitalocean_host(resource, tfvars=None): raw_attrs = resource['primary']['attributes'] name = raw_attrs['name'] @@ -189,7 +189,7 @@ def digitalocean_host(resource, tfvars=None): @parses('softlayer_virtualserver') -@calculate_mi_vars +@calculate_mantl_vars def softlayer_host(resource, module_name): raw_attrs = resource['primary']['attributes'] name = raw_attrs['name'] @@ -227,7 +227,7 @@ def softlayer_host(resource, module_name): @parses('openstack_compute_instance_v2') -@calculate_mi_vars +@calculate_mantl_vars def openstack_host(resource, module_name): raw_attrs = resource['primary']['attributes'] name = raw_attrs['name'] @@ -295,7 +295,7 @@ def openstack_host(resource, module_name): @parses('aws_instance') -@calculate_mi_vars +@calculate_mantl_vars def aws_host(resource, module_name): name = resource['primary']['attributes']['tags.Name'] raw_attrs = resource['primary']['attributes'] @@ -364,7 +364,7 @@ def aws_host(resource, module_name): @parses('google_compute_instance') -@calculate_mi_vars +@calculate_mantl_vars def gce_host(resource, module_name): name = resource['primary']['id'] raw_attrs = resource['primary']['attributes'] @@ -439,7 +439,7 @@ def gce_host(resource, module_name): @parses('vsphere_virtual_machine') -@calculate_mi_vars +@calculate_mantl_vars def vsphere_host(resource, module_name): raw_attrs = resource['primary']['attributes'] network_attrs = parse_dict(raw_attrs, 'network_interface') @@ -481,7 +481,7 @@ def vsphere_host(resource, module_name): return name, attrs, groups @parses('azure_instance') -@calculate_mi_vars +@calculate_mantl_vars def azure_host(resource, module_name): name = resource['primary']['attributes']['name'] raw_attrs = resource['primary']['attributes'] @@ -512,19 +512,19 @@ def azure_host(resource, module_name): 'ansible_ssh_host': raw_attrs['vip_address'], } - # attrs specific to microservices-infrastructure + # attrs specific to mantl attrs.update({ 'consul_dc': attrs['location'].lower().replace(" ", "-"), 'role': attrs['description'] }) - # groups specific to microservices-infrastructure + # groups specific to mantl groups.extend(['azure_image=' + attrs['image'], 'azure_location=' + attrs['location'].lower().replace(" ", "-"), 'azure_username=' + attrs['username'], 'azure_security_group=' + attrs['security_group']]) - # groups specific to microservices-infrastructure + # groups specific to mantl groups.append('role=' + attrs['role']) groups.append('dc=' + attrs['consul_dc']) From 1b44e876abe720e2e8ca58f305233560c2e60d55 Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Thu, 18 Feb 2016 15:31:10 +0100 Subject: [PATCH 3/9] tests(gce): MI -> Mantl --- tests/test_gce.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_gce.py b/tests/test_gce.py index 9b20cb0..817a82b 100644 --- a/tests/test_gce.py +++ b/tests/test_gce.py @@ -17,7 +17,7 @@ def gce_resource(): "id": "mi-control-01", "attributes": { "can_ip_forward": "false", - "description": "microservices-infrastructure control node #01", + "description": "mantl control node #01", "disk.#": "1", "disk.0.auto_delete": "true", "disk.0.device_name": "", @@ -40,7 +40,7 @@ def gce_resource(): "network_interface.0.access_config.0.nat_ip": "104.197.63.156", "network_interface.0.address": "10.0.237.130", "network_interface.0.name": "nic0", - "network_interface.0.network": "microservices-infrastructure", + "network_interface.0.network": "mantl", "self_link": "https://www.googleapis.com/compute/v1/projects/test-project/zones/us-central1-a/instances/mi-control-01", "service_account.#": "0", @@ -78,7 +78,7 @@ def test_name(gce_resource, gce_host): 'sshKeys': 'fake ssh key', }, 'network': [], 'network_interface': [{ - 'network': 'microservices-infrastructure', + 'network': 'mantl', 'access_config': [{'nat_ip': '104.197.63.156'}], 'address': '10.0.237.130', 'name': 'nic0' From 8015710755412c4c4e4d1d53df4fe15ca52a3853 Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Wed, 30 Mar 2016 09:09:30 +0100 Subject: [PATCH 4/9] Make DigitalOcean droplets without userdata work. --- terraform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform.py b/terraform.py index cab214d..bd53589 100755 --- a/terraform.py +++ b/terraform.py @@ -150,7 +150,7 @@ def digitalocean_host(resource, tfvars=None): 'image': raw_attrs['image'], 'ipv4_address': raw_attrs['ipv4_address'], 'locked': parse_bool(raw_attrs['locked']), - 'metadata': json.loads(raw_attrs['user_data']), + 'metadata': json.loads(raw_attrs.get('user_data', '{}')), 'region': raw_attrs['region'], 'size': raw_attrs['size'], 'ssh_keys': parse_list(raw_attrs, 'ssh_keys'), From 854d776e1652ac8264e1fc14ce93a658600c96bf Mon Sep 17 00:00:00 2001 From: Bekir Dogan Date: Thu, 31 Mar 2016 04:12:12 +0100 Subject: [PATCH 5/9] Export right private address for DigitalOcean. --- terraform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform.py b/terraform.py index cab214d..406592a 100755 --- a/terraform.py +++ b/terraform.py @@ -161,7 +161,8 @@ def digitalocean_host(resource, tfvars=None): 'ansible_ssh_user': 'root', # it's always "root" on DO # generic 'public_ipv4': raw_attrs['ipv4_address'], - 'private_ipv4': raw_attrs['ipv4_address'], + 'private_ipv4': raw_attrs.get('ipv4_address_private', + raw_attrs['ipv4_address']), 'provider': 'digitalocean', } From 1aa7ce8bc48387cd91d75521e48047216aee29af Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Wed, 27 Apr 2016 14:09:26 -0500 Subject: [PATCH 6/9] terraform: fix broken tests --- terraform.py | 2 +- tests/test_common.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform.py b/terraform.py index 461fdae..0ea856d 100755 --- a/terraform.py +++ b/terraform.py @@ -533,7 +533,7 @@ def azure_host(resource, module_name): @parses('clc_server') -@calculate_mi_vars +@calculate_mantl_vars def clc_server(resource, module_name): raw_attrs = resource['primary']['attributes'] name = raw_attrs.get('id') diff --git a/tests/test_common.py b/tests/test_common.py index 2a33504..1b2b654 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -3,9 +3,9 @@ @pytest.fixture -def calculate_mi_vars(): - from terraform import calculate_mi_vars - return calculate_mi_vars +def calculate_mantl_vars(): + from terraform import calculate_mantl_vars + return calculate_mantl_vars def mirror(item): @@ -18,22 +18,22 @@ def inner(*args, **kwargs): @pytest.mark.parametrize('role,serverstate', [('control', True), ('worker', False), ('none', False)]) -def test_attrs(calculate_mi_vars, role, serverstate): +def test_attrs(calculate_mantl_vars, role, serverstate): mirrorer = mirror(('', {'role': role}, [])) - func = calculate_mi_vars(mirrorer) + func = calculate_mantl_vars(mirrorer) _, attrs, _ = func() assert 'consul_is_server' in attrs assert attrs['consul_is_server'] == serverstate @pytest.mark.parametrize('routable', [True, False]) -def test_publicly_routable(calculate_mi_vars, routable): +def test_publicly_routable(calculate_mantl_vars, routable): attrs = {} if routable: attrs['publicly_routable'] = True mirrorer = mirror(('', attrs, [])) - func = calculate_mi_vars(mirrorer) + func = calculate_mantl_vars(mirrorer) _, _, groups = func() if routable: assert 'publicly_routable' in groups From 714c86e9c6a1e104747bfe27e54e7668bfa59d4e Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Wed, 27 Apr 2016 14:44:08 -0500 Subject: [PATCH 7/9] terraform: add triton --- terraform.py | 70 +++++++++++++++++++++++++++++++ tests/test_triton.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 tests/test_triton.py diff --git a/terraform.py b/terraform.py index 0ea856d..ac8aee9 100755 --- a/terraform.py +++ b/terraform.py @@ -138,6 +138,75 @@ def parse_bool(string_form): raise ValueError('could not convert %r to a bool' % string_form) +@parses('triton_machine') +@calculate_mantl_vars +def triton_machine(resource, module_name): + raw_attrs = resource['primary']['attributes'] + name = raw_attrs.get('name') + groups = [] + + attrs = { + 'id': raw_attrs['id'], + 'dataset': raw_attrs['dataset'], + 'disk': raw_attrs['disk'], + 'firewall_enabled': parse_bool(raw_attrs['firewall_enabled']), + 'image': raw_attrs['image'], + 'ips': parse_list(raw_attrs, 'ips'), + 'memory': raw_attrs['memory'], + 'name': raw_attrs['name'], + 'networks': parse_list(raw_attrs, 'networks'), + 'package': raw_attrs['package'], + 'primary_ip': raw_attrs['primaryip'], + 'root_authorized_keys': raw_attrs['root_authorized_keys'], + 'state': raw_attrs['state'], + 'tags': parse_dict(raw_attrs, 'tags'), + 'type': raw_attrs['type'], + 'user_data': raw_attrs['user_data'], + 'user_script': raw_attrs['user_script'], + + # ansible + 'ansible_ssh_host': raw_attrs['primaryip'], + 'ansible_ssh_port': 22, + 'ansible_ssh_user': 'root', # it's "root" on Triton by default + + # generic + 'public_ipv4': raw_attrs['primaryip'], + 'provider': 'triton', + } + + # private IPv4 + for ip in attrs['ips']: + if ip.startswith('10') or ip.startswith('192.168'): # private IPs + attrs['private_ipv4'] = ip + break + + if 'private_ipv4' not in attrs: + attrs['private_ipv4'] = attrs['public_ipv4'] + + # attrs specific to Mantl + attrs.update({ + 'consul_dc': _clean_dc(attrs['tags'].get('dc', 'none')), + 'role': attrs['tags'].get('role', 'none'), + 'ansible_python_interpreter': attrs['tags'].get('python_bin', 'python') + }) + + # add groups based on attrs + groups.append('triton_image=' + attrs['image']) + groups.append('triton_package=' + attrs['package']) + groups.append('triton_state=' + attrs['state']) + groups.append('triton_firewall_enabled=%s' % attrs['firewall_enabled']) + groups.extend('triton_tags_%s=%s' % item + for item in attrs['tags'].items()) + groups.extend('triton_network=%s' + network + for network in attrs['networks']) + + # groups specific to Mantl + groups.append('role=' + attrs['role']) + groups.append('dc=' + attrs['consul_dc']) + + return name, attrs, groups + + @parses('digitalocean_droplet') @calculate_mantl_vars def digitalocean_host(resource, tfvars=None): @@ -570,6 +639,7 @@ def clc_server(resource, module_name): return name, attrs, groups + ## QUERY TYPES def query_host(hosts, target): for name, attrs, _ in hosts: diff --git a/tests/test_triton.py b/tests/test_triton.py new file mode 100644 index 0000000..83c51ad --- /dev/null +++ b/tests/test_triton.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +import pytest + + +@pytest.fixture +def triton_machine(): + from terraform import triton_machine + return triton_machine + + +@pytest.fixture +def triton_resource(): + return { + "type": "triton_machine", + "primary": { + "id": "69818186-01ed-4f5b-af1b-ce9c23e16f82", + "attributes": { + "administrator_pw": "", + "created": "2016-04-27T18:57:49.189Z", + "dataset": "", + "disk": "122880", + "firewall_enabled": "false", + "id": "69818186-01ed-4f5b-af1b-ce9c23e16f82", + "image": "dd31507e-031e-11e6-be8a-8f2707b5b3ee", + "ips.#": "2", + "ips.0": "165.225.136.36", + "ips.1": "10.112.7.149", + "memory": "4096", + "name": "mantl-control-01", + "networks.#": "2", + "networks.0": "65ae3604-7c5c-4255-9c9f-6248e5d78900", + "networks.1": "56f0fd52-4df1-49bd-af0c-81c717ea8bce", + "package": "Medium 4GB", + "primaryip": "165.225.136.36", + "root_authorized_keys": "key text replaced for test", + "state": "running", + "tags.#": "1", + "tags.role": "control", + "type": "virtualmachine", + "updated": "2016-04-27T18:58:05.000Z", + "user_data": "", + "user_script": "" + } + } + } + + +def test_name(triton_resource, triton_machine): + name, _, _ = triton_machine(triton_resource, '') + assert name == 'mantl-control-01' + + +@pytest.mark.parametrize('attr,should', { + 'id': '69818186-01ed-4f5b-af1b-ce9c23e16f82', + 'dataset': '', + 'disk': '122880', + 'firewall_enabled': False, + 'image': 'dd31507e-031e-11e6-be8a-8f2707b5b3ee', + 'ips': ['10.112.7.149', '165.225.136.36'], + 'name': 'mantl-control-01', + 'networks': ['56f0fd52-4df1-49bd-af0c-81c717ea8bce', '65ae3604-7c5c-4255-9c9f-6248e5d78900'], + 'package': 'Medium 4GB', + 'primary_ip': '165.225.136.36', + 'root_authorized_keys': 'key text replaced for test', + 'state': 'running', + 'tags': {'role': 'control'}, + 'type': 'virtualmachine', + 'user_data': '', + 'user_script': '', + # ansible + 'ansible_ssh_host': '165.225.136.36', + 'ansible_ssh_port': 22, + 'ansible_ssh_user': 'root', + # generic + 'public_ipv4': '165.225.136.36', + 'private_ipv4': '10.112.7.149', + 'provider': 'triton', + # mi + 'consul_dc': 'none', + 'role': 'control', +}.items()) +def test_attrs(triton_resource, triton_machine, attr, should): + _, attrs, _ = triton_machine(triton_resource, 'module_name') + assert attr in attrs + assert attrs[attr] == should + + +@pytest.mark.parametrize( + 'group', ['triton_image=dd31507e-031e-11e6-be8a-8f2707b5b3ee', + 'triton_package=Medium 4GB', 'triton_state=running', + 'triton_firewall_enabled=False', 'triton_tags_role=control', + 'triton_network=%s56f0fd52-4df1-49bd-af0c-81c717ea8bce', + 'triton_network=%s65ae3604-7c5c-4255-9c9f-6248e5d78900', + 'role=control', 'dc=none'], +) +def test_groups(triton_resource, triton_machine, group): + _, _, groups = triton_machine(triton_resource, 'module_name') + print groups + assert group in groups From 83aa54ec3dc108d3ce38ee4d53e45510cbf9a379 Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Wed, 27 Apr 2016 14:46:43 -0500 Subject: [PATCH 8/9] travis: remove pypy tests --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ef429d..64dafe7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - 2.7 - - pypy install: "pip install -r requirements.txt" -script: py.test \ No newline at end of file +script: py.test From a2a73d2fe80e6ac4453485f6f62dc7c6772f1680 Mon Sep 17 00:00:00 2001 From: Brian Hicks Date: Wed, 27 Apr 2016 15:23:27 -0500 Subject: [PATCH 9/9] terraform(triton): remove stray %s --- terraform.py | 2 +- tests/test_triton.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform.py b/terraform.py index ac8aee9..82278f2 100755 --- a/terraform.py +++ b/terraform.py @@ -197,7 +197,7 @@ def triton_machine(resource, module_name): groups.append('triton_firewall_enabled=%s' % attrs['firewall_enabled']) groups.extend('triton_tags_%s=%s' % item for item in attrs['tags'].items()) - groups.extend('triton_network=%s' + network + groups.extend('triton_network=' + network for network in attrs['networks']) # groups specific to Mantl diff --git a/tests/test_triton.py b/tests/test_triton.py index 83c51ad..14c2a56 100644 --- a/tests/test_triton.py +++ b/tests/test_triton.py @@ -89,8 +89,8 @@ def test_attrs(triton_resource, triton_machine, attr, should): 'group', ['triton_image=dd31507e-031e-11e6-be8a-8f2707b5b3ee', 'triton_package=Medium 4GB', 'triton_state=running', 'triton_firewall_enabled=False', 'triton_tags_role=control', - 'triton_network=%s56f0fd52-4df1-49bd-af0c-81c717ea8bce', - 'triton_network=%s65ae3604-7c5c-4255-9c9f-6248e5d78900', + 'triton_network=56f0fd52-4df1-49bd-af0c-81c717ea8bce', + 'triton_network=65ae3604-7c5c-4255-9c9f-6248e5d78900', 'role=control', 'dc=none'], ) def test_groups(triton_resource, triton_machine, group):