From 32a16dc6f2ec69b95cc4ea5c31ff488d9e47d0c2 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 19:52:33 +0000 Subject: [PATCH 01/13] Improved the Python API example Use the new default for most distro (Python3), make it OO-based rather than sequentially scripted, make it runnable, improve API, etc. --- python/example.py | 145 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 102 insertions(+), 43 deletions(-) diff --git a/python/example.py b/python/example.py index db80738..2e6b256 100644 --- a/python/example.py +++ b/python/example.py @@ -1,60 +1,119 @@ -import httplib +#!/usr/bin/env python3 + +import http.client +import json import os import ssl import tempfile -# NOTE - to run this example, you need at least python 2.7.9 +class FlockerApi(object): + DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') + + def __init__(self, api_version = 1): + control_service = os.environ.get("CONTROL_SERVICE", "localhost") + control_port = os.environ.get("CONTROL_PORT", 4523) + + self._api_version = api_version + + key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) + cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) + ca_file = os.environ.get("CA_FILE", "%s/cluster.crt" % self.DEFAULT_PLUGIN_DIR) + + # Create a certificate chain and then pass that into the SSL system. + cert_with_chain_tempfile = tempfile.NamedTemporaryFile() + + temp_cert_with_chain_path = cert_with_chain_tempfile.name + os.chmod(temp_cert_with_chain_path, 0o0600) + + # Write our cert and append the CA cert to build the chain + with open(cert_file, 'rb') as cert_file_obj: + cert_with_chain_tempfile.write(cert_file_obj.read()) + + cert_with_chain_tempfile.write('\n'.encode('utf-8')) + + with open(ca_file, 'rb') as cacert_file_obj: + cert_with_chain_tempfile.write(cacert_file_obj.read()) + + # Reset file pointer for the SSL context to read it properly + cert_with_chain_tempfile.seek(0) + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ssl_context.load_cert_chain(temp_cert_with_chain_path, key_file) + + self._http_client = http.client.HTTPSConnection(control_service, + control_port, + context=ssl_context) + + # XXX: These should really be generic functions created dynamically + def get(self, endpoint, data = None): + return self._make_api_request('GET', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def post(self, endpoint, data = None): + return self._make_api_request('POST', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def delete(self, endpoint, data = None): + return self._make_api_request('DELETE', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def _make_api_request(self, method, endpoint, data = None): + # Convert data to string if it's not yet in this format + if data and not isinstance(data, str): + data = json.dumps(data).encode('utf-8') + + headers = {"Content-type": "application/json"} + self._http_client.request(method, endpoint, data, + headers=headers) -# Define the control IP, port, and the certificates for authentication. + response = self._http_client.getresponse() -CONTROL_SERVICE = os.environ.get( - "CONTROL_SERVICE", "54.157.8.57") -CONTROL_PORT = os.environ.get( - "CONTROL_PORT", 4523) -KEY_FILE = os.environ.get( - "KEY_FILE", "/Users/kai/projects/flocker-api-examples/flockerdemo.key") -CERT_FILE = os.environ.get( - "CERT_FILE", "/Users/kai/projects/flocker-api-examples/flockerdemo.crt") -CA_FILE = os.environ.get( - "CA_FILE", "/Users/kai/projects/flocker-api-examples/cluster.crt") + status = response.status + body = response.read() -# Create a certificate chain and then pass that into the SSL system. + print('Status:', status) -certtemp = tempfile.NamedTemporaryFile() -TEMP_CERT_CA_FILE = certtemp.name -os.chmod(TEMP_CERT_CA_FILE, 0600) -certtemp.write(open(CERT_FILE).read()) -certtemp.write("\n") -certtemp.write(open(CA_FILE).read()) -certtemp.seek(0) -ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) -ctx.load_cert_chain(TEMP_CERT_CA_FILE, KEY_FILE) + # If you want debugging + # print('Body:', body) -# Finally, create a HTTP connection. + print() -c = httplib.HTTPSConnection(CONTROL_SERVICE, CONTROL_PORT, context=ctx) + return json.loads(body.decode('utf-8')) -def make_api_request(method, endpoint, data=None): - if method in ("GET", "DELETE"): - c.request(method, endpoint) - elif method == "POST": - c.request("POST", endpoint, data, - headers={"Content-type": "application/json"}) - else: - raise Exception("Unknown method %s" % (method,)) +if __name__ == '__main__': + api = FlockerApi() - r = c.getresponse() - body = r.read() - status = r.status + # Show us the version of Flocker + version = api.get('version') + print("Version:", version['flocker']) - print body + # Get current volumes (datasets) + print('Datasets:') + datasets = api.get('configuration/datasets') + print(json.dumps(datasets, sort_keys=True, indent=4)) -# Make the first request to check the service is working. + # Create a Flocker volume of size 10GB + size_in_gb = 10 -make_api_request("GET", "/v1/version") + print('Trying to reuse the primary from returned list') + primary_id = datasets[0]['primary'] + print('Primary:', primary_id) -# Create a volume. + # XXX: Using shifts to evaluate max bytes + # '<< 30' == '* 2^30' == '* 1024 * 1024 * 1024' + data = { + 'primary': primary_id, + 'maximum_size': size_in_gb << 30, + 'metadata': { +# If your backend supports profiles uncomment the following: +# 'clusterhq:flocker:profile': 'silver', + 'name': 'my-test-volume3' + } + } -make_api_request("POST", "/v1/configuration/datasets", - data= r'{"primary": "%s", "maximum_size": 107374182400, "metadata": {"name": "example_dataset"}}' - % ("5540d6e3-392b-4da0-828a-34b724c5bb80",)) \ No newline at end of file + print('Create volume:') + dataset_create_result = api.post('configuration/datasets', data) + print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) From eb2349f6e00dd689a70af204242d19b08b136f64 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 19:53:44 +0000 Subject: [PATCH 02/13] Fix Dockerfile for Python example There was no way this could work before without manual volume mounting and now we use Python3 as well. --- python/Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/Dockerfile b/python/Dockerfile index 58d86a4..6778865 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,3 +1,9 @@ -FROM python:2-onbuild +FROM python:3.4 + MAINTAINER Kai Davenport -CMD [ "python", "./example.py" ] \ No newline at end of file +MAINTAINER Srdjan Grubor + +ADD ./example.py . +RUN chmod +x ./example.py + +CMD [ "./example.py" ] From fdf9d3974c6efe4770710d148380b0390579b404 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:13:43 +0000 Subject: [PATCH 03/13] Copied example.py to flocker_api.py to create a REST client This will be the basis for creating a simple REST API client for flocker since other tools are either more complicated or have no good way to install them. --- python/flocker_api.py | 119 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 python/flocker_api.py diff --git a/python/flocker_api.py b/python/flocker_api.py new file mode 100755 index 0000000..2e6b256 --- /dev/null +++ b/python/flocker_api.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +import http.client +import json +import os +import ssl +import tempfile + +class FlockerApi(object): + DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') + + def __init__(self, api_version = 1): + control_service = os.environ.get("CONTROL_SERVICE", "localhost") + control_port = os.environ.get("CONTROL_PORT", 4523) + + self._api_version = api_version + + key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) + cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) + ca_file = os.environ.get("CA_FILE", "%s/cluster.crt" % self.DEFAULT_PLUGIN_DIR) + + # Create a certificate chain and then pass that into the SSL system. + cert_with_chain_tempfile = tempfile.NamedTemporaryFile() + + temp_cert_with_chain_path = cert_with_chain_tempfile.name + os.chmod(temp_cert_with_chain_path, 0o0600) + + # Write our cert and append the CA cert to build the chain + with open(cert_file, 'rb') as cert_file_obj: + cert_with_chain_tempfile.write(cert_file_obj.read()) + + cert_with_chain_tempfile.write('\n'.encode('utf-8')) + + with open(ca_file, 'rb') as cacert_file_obj: + cert_with_chain_tempfile.write(cacert_file_obj.read()) + + # Reset file pointer for the SSL context to read it properly + cert_with_chain_tempfile.seek(0) + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ssl_context.load_cert_chain(temp_cert_with_chain_path, key_file) + + self._http_client = http.client.HTTPSConnection(control_service, + control_port, + context=ssl_context) + + # XXX: These should really be generic functions created dynamically + def get(self, endpoint, data = None): + return self._make_api_request('GET', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def post(self, endpoint, data = None): + return self._make_api_request('POST', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def delete(self, endpoint, data = None): + return self._make_api_request('DELETE', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def _make_api_request(self, method, endpoint, data = None): + # Convert data to string if it's not yet in this format + if data and not isinstance(data, str): + data = json.dumps(data).encode('utf-8') + + headers = {"Content-type": "application/json"} + self._http_client.request(method, endpoint, data, + headers=headers) + + response = self._http_client.getresponse() + + status = response.status + body = response.read() + + print('Status:', status) + + # If you want debugging + # print('Body:', body) + + print() + + return json.loads(body.decode('utf-8')) + +if __name__ == '__main__': + api = FlockerApi() + + # Show us the version of Flocker + version = api.get('version') + print("Version:", version['flocker']) + + # Get current volumes (datasets) + print('Datasets:') + datasets = api.get('configuration/datasets') + print(json.dumps(datasets, sort_keys=True, indent=4)) + + # Create a Flocker volume of size 10GB + size_in_gb = 10 + + print('Trying to reuse the primary from returned list') + primary_id = datasets[0]['primary'] + print('Primary:', primary_id) + + # XXX: Using shifts to evaluate max bytes + # '<< 30' == '* 2^30' == '* 1024 * 1024 * 1024' + data = { + 'primary': primary_id, + 'maximum_size': size_in_gb << 30, + 'metadata': { +# If your backend supports profiles uncomment the following: +# 'clusterhq:flocker:profile': 'silver', + 'name': 'my-test-volume3' + } + } + + print('Create volume:') + dataset_create_result = api.post('configuration/datasets', data) + print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) From 266532d6e121312435abb7015f666afb177ccf9f Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:33:34 +0000 Subject: [PATCH 04/13] Added ability to delete volumes easily to flocker_api This should make things easier when reusing this code to create volumes. --- python/flocker_api.py | 82 ++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index 2e6b256..9430b3c 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -60,6 +60,32 @@ def delete(self, endpoint, data = None): "/v%s/%s" % (self._api_version, endpoint), data) + # Specific API requests + def get_version(self): + version = self.get('version') + return version['flocker'] + + def create_volume(self, name, size_in_gb, primary_id, profile = None): + if not isinstance(size_in_gb, int): + print('Error! Size must be an integer!') + exit(1) + + data = { + 'primary': primary_id, + 'maximum_size': size_in_gb << 30, + 'metadata': { + 'name': name + } + } + + if profile: + data['metadata']['clusterhq:flocker:profile'] = profile + + return api.post('configuration/datasets', data) + + def delete_volume(self, dataset_id): + return self.delete('configuration/datasets/%s' % dataset_id) + def _make_api_request(self, method, endpoint, data = None): # Convert data to string if it's not yet in this format if data and not isinstance(data, str): @@ -84,36 +110,26 @@ def _make_api_request(self, method, endpoint, data = None): return json.loads(body.decode('utf-8')) if __name__ == '__main__': - api = FlockerApi() - - # Show us the version of Flocker - version = api.get('version') - print("Version:", version['flocker']) - - # Get current volumes (datasets) - print('Datasets:') - datasets = api.get('configuration/datasets') - print(json.dumps(datasets, sort_keys=True, indent=4)) - - # Create a Flocker volume of size 10GB - size_in_gb = 10 - - print('Trying to reuse the primary from returned list') - primary_id = datasets[0]['primary'] - print('Primary:', primary_id) - - # XXX: Using shifts to evaluate max bytes - # '<< 30' == '* 2^30' == '* 1024 * 1024 * 1024' - data = { - 'primary': primary_id, - 'maximum_size': size_in_gb << 30, - 'metadata': { -# If your backend supports profiles uncomment the following: -# 'clusterhq:flocker:profile': 'silver', - 'name': 'my-test-volume3' - } - } - - print('Create volume:') - dataset_create_result = api.post('configuration/datasets', data) - print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) + api = FlockerApi() + + # Show us the version of Flocker + print("Version:", api.get_version()) + + # Get current volumes (datasets) + print('Datasets:') + datasets = api.get('configuration/datasets') + print(json.dumps(datasets, sort_keys=True, indent=4)) + + + print('Trying to reuse the primary from returned list') + primary_id = datasets[0]['primary'] + print('Primary:', primary_id) + + print('Create volume:') + # Create a Flocker volume of size 10GB + dataset_create_result = api.create_volume('my-test-volume3', 10, primary_id, profile = "gold") + print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) + + volume_id = dataset_create_result['dataset_id'] + delete_result = api.delete_volume(volume_id) + print(json.dumps(delete_result, sort_keys=True, indent=4)) From 6ab9ec083702f14c88a1ecaa303c5f646d99a897 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:36:07 +0000 Subject: [PATCH 05/13] Made get_volume a method for flocker_api script We can now get this data without specifying the endpoint --- python/flocker_api.py | 51 +++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index 9430b3c..fa00ae9 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -60,6 +60,29 @@ def delete(self, endpoint, data = None): "/v%s/%s" % (self._api_version, endpoint), data) + def _make_api_request(self, method, endpoint, data = None): + # Convert data to string if it's not yet in this format + if data and not isinstance(data, str): + data = json.dumps(data).encode('utf-8') + + headers = {"Content-type": "application/json"} + self._http_client.request(method, endpoint, data, + headers=headers) + + response = self._http_client.getresponse() + + status = response.status + body = response.read() + + print('Status:', status) + + # If you want debugging + # print('Body:', body) + + print() + + return json.loads(body.decode('utf-8')) + # Specific API requests def get_version(self): version = self.get('version') @@ -84,30 +107,10 @@ def create_volume(self, name, size_in_gb, primary_id, profile = None): return api.post('configuration/datasets', data) def delete_volume(self, dataset_id): - return self.delete('configuration/datasets/%s' % dataset_id) + return self.delete('configuration/datasets/%s' % dataset_id) - def _make_api_request(self, method, endpoint, data = None): - # Convert data to string if it's not yet in this format - if data and not isinstance(data, str): - data = json.dumps(data).encode('utf-8') - - headers = {"Content-type": "application/json"} - self._http_client.request(method, endpoint, data, - headers=headers) - - response = self._http_client.getresponse() - - status = response.status - body = response.read() - - print('Status:', status) - - # If you want debugging - # print('Body:', body) - - print() - - return json.loads(body.decode('utf-8')) + def get_volumes(self): + return api.get('configuration/datasets') if __name__ == '__main__': api = FlockerApi() @@ -117,7 +120,7 @@ def _make_api_request(self, method, endpoint, data = None): # Get current volumes (datasets) print('Datasets:') - datasets = api.get('configuration/datasets') + datasets = api.get_volumes() print(json.dumps(datasets, sort_keys=True, indent=4)) From 009bfc078ea26674c6a1d2b809e77c74104e01b5 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:42:09 +0000 Subject: [PATCH 06/13] Added use of 'X-If-Configuration-Matches' headers to flocker_api We need this to ensure consistency of the cluster. --- python/flocker_api.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index fa00ae9..34d1a23 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -14,6 +14,7 @@ def __init__(self, api_version = 1): control_port = os.environ.get("CONTROL_PORT", 4523) self._api_version = api_version + self._last_known_config = None key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) @@ -65,7 +66,10 @@ def _make_api_request(self, method, endpoint, data = None): if data and not isinstance(data, str): data = json.dumps(data).encode('utf-8') - headers = {"Content-type": "application/json"} + headers = { 'Content-type': 'application/json' } + if self._last_known_config: + headers['X-If-Configuration-Matches'] = self._last_known_config + self._http_client.request(method, endpoint, data, headers=headers) @@ -74,6 +78,10 @@ def _make_api_request(self, method, endpoint, data = None): status = response.status body = response.read() + # Make sure to use the X + if 'X-Configuration-Tag' in response.getheaders(): + self._last_known_config = response.getheaders()['X-Configuration-Tag'].decode('utf-8') + print('Status:', status) # If you want debugging From 152d20cdc88509bf9980990bff0eb45026fb5546 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:54:09 +0000 Subject: [PATCH 07/13] Added move_volume to flocker_api script This allows the api to now easily moves volumes around on nodes. --- python/flocker_api.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index 34d1a23..b0c635a 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -9,11 +9,12 @@ class FlockerApi(object): DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') - def __init__(self, api_version = 1): + def __init__(self, api_version = 1, debug = False): control_service = os.environ.get("CONTROL_SERVICE", "localhost") control_port = os.environ.get("CONTROL_PORT", 4523) self._api_version = api_version + self._debug = debug self._last_known_config = None key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) @@ -84,12 +85,17 @@ def _make_api_request(self, method, endpoint, data = None): print('Status:', status) - # If you want debugging + # If you want verbose debugging # print('Body:', body) print() - return json.loads(body.decode('utf-8')) + result = json.loads(body.decode('utf-8')) + + if self._debug == True: + print(json.dumps(result, sort_keys=True, indent=4)) + + return result # Specific API requests def get_version(self): @@ -114,14 +120,21 @@ def create_volume(self, name, size_in_gb, primary_id, profile = None): return api.post('configuration/datasets', data) + def move_volume(self, volume_id, new_primary_id): + data = { 'primary': new_primary_id } + return api.post('configuration/datasets/%s' % volume_id, data) + def delete_volume(self, dataset_id): return self.delete('configuration/datasets/%s' % dataset_id) def get_volumes(self): return api.get('configuration/datasets') + def get_nodes(self): + return api.get('state/nodes') + if __name__ == '__main__': - api = FlockerApi() + api = FlockerApi(debug = True) # Show us the version of Flocker print("Version:", api.get_version()) @@ -129,8 +142,9 @@ def get_volumes(self): # Get current volumes (datasets) print('Datasets:') datasets = api.get_volumes() - print(json.dumps(datasets, sort_keys=True, indent=4)) + print('Nodes:') + nodes = api.get_nodes() print('Trying to reuse the primary from returned list') primary_id = datasets[0]['primary'] @@ -139,8 +153,10 @@ def get_volumes(self): print('Create volume:') # Create a Flocker volume of size 10GB dataset_create_result = api.create_volume('my-test-volume3', 10, primary_id, profile = "gold") - print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) - volume_id = dataset_create_result['dataset_id'] + + print('Move volume (to the same node):') + move_result = api.move_volume(volume_id, primary_id) + + print('Delete volume:') delete_result = api.delete_volume(volume_id) - print(json.dumps(delete_result, sort_keys=True, indent=4)) From 09b231b3abe15785733427e8751612cb6aa04bfb Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 20:56:11 +0000 Subject: [PATCH 08/13] Added ability to get_leases from flocker_api script This also needed to be added as there was no easy way to get it without hardcoded strings. --- python/flocker_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/flocker_api.py b/python/flocker_api.py index b0c635a..81ad99e 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -133,6 +133,9 @@ def get_volumes(self): def get_nodes(self): return api.get('state/nodes') + def get_leases(self): + return api.get('configuration/leases') + if __name__ == '__main__': api = FlockerApi(debug = True) @@ -146,6 +149,9 @@ def get_nodes(self): print('Nodes:') nodes = api.get_nodes() + print('Leases:') + leases = api.get_leases() + print('Trying to reuse the primary from returned list') primary_id = datasets[0]['primary'] print('Primary:', primary_id) From 7764079c450cefac2634d187ab6b48fc8a1557c3 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 21:06:36 +0000 Subject: [PATCH 09/13] Added ability of creating/releasing leases to python example This should round out the API for REST. --- python/flocker_api.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index 81ad99e..a2988b4 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -118,23 +118,32 @@ def create_volume(self, name, size_in_gb, primary_id, profile = None): if profile: data['metadata']['clusterhq:flocker:profile'] = profile - return api.post('configuration/datasets', data) + return self.post('configuration/datasets', data) def move_volume(self, volume_id, new_primary_id): data = { 'primary': new_primary_id } - return api.post('configuration/datasets/%s' % volume_id, data) + return self.post('configuration/datasets/%s' % volume_id, data) def delete_volume(self, dataset_id): return self.delete('configuration/datasets/%s' % dataset_id) def get_volumes(self): - return api.get('configuration/datasets') + return self.get('configuration/datasets') def get_nodes(self): - return api.get('state/nodes') + return self.get('state/nodes') def get_leases(self): - return api.get('configuration/leases') + return self.get('configuration/leases') + + def release_lease(self, dataset_id): + return self.delete('configuration/leases/%s' % dataset_id) + + def acquire_lease(self, dataset_id, node_id, expires = None): + data = { 'dataset_id': dataset_id, + 'node_uuid': node_id, + 'expires': expires } + return self.post('configuration/leases', data) if __name__ == '__main__': api = FlockerApi(debug = True) @@ -148,9 +157,7 @@ def get_leases(self): print('Nodes:') nodes = api.get_nodes() - - print('Leases:') - leases = api.get_leases() + first_node = nodes[0]['uuid'] print('Trying to reuse the primary from returned list') primary_id = datasets[0]['primary'] @@ -158,11 +165,24 @@ def get_leases(self): print('Create volume:') # Create a Flocker volume of size 10GB - dataset_create_result = api.create_volume('my-test-volume3', 10, primary_id, profile = "gold") + dataset_create_result = api.create_volume('my-test-volume3', + 10, + primary_id, + profile = "gold") volume_id = dataset_create_result['dataset_id'] print('Move volume (to the same node):') - move_result = api.move_volume(volume_id, primary_id) + api.move_volume(volume_id, primary_id) + + print('Acquire lease:') + api.acquire_lease(volume_id, first_node) + + print('Leases:') + leases = api.get_leases() + lease_id = leases[0] + + print('Release lease:') + api.release_lease(volume_id) print('Delete volume:') - delete_result = api.delete_volume(volume_id) + api.delete_volume(volume_id) From c3710b72cb113c19d9434b9a97debe9ec65a7b94 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 21:07:37 +0000 Subject: [PATCH 10/13] Moved flocker_api to example.py Since we don't do any CLI processing, this is effectively still just an example of API usage. --- python/example.py | 143 +++++++++++++++++++++++--------- python/flocker_api.py | 188 ------------------------------------------ 2 files changed, 106 insertions(+), 225 deletions(-) mode change 100644 => 100755 python/example.py delete mode 100755 python/flocker_api.py diff --git a/python/example.py b/python/example.py old mode 100644 new mode 100755 index 2e6b256..a2988b4 --- a/python/example.py +++ b/python/example.py @@ -9,11 +9,13 @@ class FlockerApi(object): DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') - def __init__(self, api_version = 1): + def __init__(self, api_version = 1, debug = False): control_service = os.environ.get("CONTROL_SERVICE", "localhost") control_port = os.environ.get("CONTROL_PORT", 4523) self._api_version = api_version + self._debug = debug + self._last_known_config = None key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) @@ -65,7 +67,10 @@ def _make_api_request(self, method, endpoint, data = None): if data and not isinstance(data, str): data = json.dumps(data).encode('utf-8') - headers = {"Content-type": "application/json"} + headers = { 'Content-type': 'application/json' } + if self._last_known_config: + headers['X-If-Configuration-Matches'] = self._last_known_config + self._http_client.request(method, endpoint, data, headers=headers) @@ -74,46 +79,110 @@ def _make_api_request(self, method, endpoint, data = None): status = response.status body = response.read() + # Make sure to use the X + if 'X-Configuration-Tag' in response.getheaders(): + self._last_known_config = response.getheaders()['X-Configuration-Tag'].decode('utf-8') + print('Status:', status) - # If you want debugging + # If you want verbose debugging # print('Body:', body) print() - return json.loads(body.decode('utf-8')) + result = json.loads(body.decode('utf-8')) + + if self._debug == True: + print(json.dumps(result, sort_keys=True, indent=4)) + + return result + + # Specific API requests + def get_version(self): + version = self.get('version') + return version['flocker'] + + def create_volume(self, name, size_in_gb, primary_id, profile = None): + if not isinstance(size_in_gb, int): + print('Error! Size must be an integer!') + exit(1) + + data = { + 'primary': primary_id, + 'maximum_size': size_in_gb << 30, + 'metadata': { + 'name': name + } + } + + if profile: + data['metadata']['clusterhq:flocker:profile'] = profile + + return self.post('configuration/datasets', data) + + def move_volume(self, volume_id, new_primary_id): + data = { 'primary': new_primary_id } + return self.post('configuration/datasets/%s' % volume_id, data) + + def delete_volume(self, dataset_id): + return self.delete('configuration/datasets/%s' % dataset_id) + + def get_volumes(self): + return self.get('configuration/datasets') + + def get_nodes(self): + return self.get('state/nodes') + + def get_leases(self): + return self.get('configuration/leases') + + def release_lease(self, dataset_id): + return self.delete('configuration/leases/%s' % dataset_id) + + def acquire_lease(self, dataset_id, node_id, expires = None): + data = { 'dataset_id': dataset_id, + 'node_uuid': node_id, + 'expires': expires } + return self.post('configuration/leases', data) if __name__ == '__main__': - api = FlockerApi() - - # Show us the version of Flocker - version = api.get('version') - print("Version:", version['flocker']) - - # Get current volumes (datasets) - print('Datasets:') - datasets = api.get('configuration/datasets') - print(json.dumps(datasets, sort_keys=True, indent=4)) - - # Create a Flocker volume of size 10GB - size_in_gb = 10 - - print('Trying to reuse the primary from returned list') - primary_id = datasets[0]['primary'] - print('Primary:', primary_id) - - # XXX: Using shifts to evaluate max bytes - # '<< 30' == '* 2^30' == '* 1024 * 1024 * 1024' - data = { - 'primary': primary_id, - 'maximum_size': size_in_gb << 30, - 'metadata': { -# If your backend supports profiles uncomment the following: -# 'clusterhq:flocker:profile': 'silver', - 'name': 'my-test-volume3' - } - } - - print('Create volume:') - dataset_create_result = api.post('configuration/datasets', data) - print(json.dumps(dataset_create_result, sort_keys=True, indent=4)) + api = FlockerApi(debug = True) + + # Show us the version of Flocker + print("Version:", api.get_version()) + + # Get current volumes (datasets) + print('Datasets:') + datasets = api.get_volumes() + + print('Nodes:') + nodes = api.get_nodes() + first_node = nodes[0]['uuid'] + + print('Trying to reuse the primary from returned list') + primary_id = datasets[0]['primary'] + print('Primary:', primary_id) + + print('Create volume:') + # Create a Flocker volume of size 10GB + dataset_create_result = api.create_volume('my-test-volume3', + 10, + primary_id, + profile = "gold") + volume_id = dataset_create_result['dataset_id'] + + print('Move volume (to the same node):') + api.move_volume(volume_id, primary_id) + + print('Acquire lease:') + api.acquire_lease(volume_id, first_node) + + print('Leases:') + leases = api.get_leases() + lease_id = leases[0] + + print('Release lease:') + api.release_lease(volume_id) + + print('Delete volume:') + api.delete_volume(volume_id) diff --git a/python/flocker_api.py b/python/flocker_api.py deleted file mode 100755 index a2988b4..0000000 --- a/python/flocker_api.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 - -import http.client -import json -import os -import ssl -import tempfile - -class FlockerApi(object): - DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') - - def __init__(self, api_version = 1, debug = False): - control_service = os.environ.get("CONTROL_SERVICE", "localhost") - control_port = os.environ.get("CONTROL_PORT", 4523) - - self._api_version = api_version - self._debug = debug - self._last_known_config = None - - key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) - cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) - ca_file = os.environ.get("CA_FILE", "%s/cluster.crt" % self.DEFAULT_PLUGIN_DIR) - - # Create a certificate chain and then pass that into the SSL system. - cert_with_chain_tempfile = tempfile.NamedTemporaryFile() - - temp_cert_with_chain_path = cert_with_chain_tempfile.name - os.chmod(temp_cert_with_chain_path, 0o0600) - - # Write our cert and append the CA cert to build the chain - with open(cert_file, 'rb') as cert_file_obj: - cert_with_chain_tempfile.write(cert_file_obj.read()) - - cert_with_chain_tempfile.write('\n'.encode('utf-8')) - - with open(ca_file, 'rb') as cacert_file_obj: - cert_with_chain_tempfile.write(cacert_file_obj.read()) - - # Reset file pointer for the SSL context to read it properly - cert_with_chain_tempfile.seek(0) - - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - ssl_context.load_cert_chain(temp_cert_with_chain_path, key_file) - - self._http_client = http.client.HTTPSConnection(control_service, - control_port, - context=ssl_context) - - # XXX: These should really be generic functions created dynamically - def get(self, endpoint, data = None): - return self._make_api_request('GET', - "/v%s/%s" % (self._api_version, endpoint), - data) - - def post(self, endpoint, data = None): - return self._make_api_request('POST', - "/v%s/%s" % (self._api_version, endpoint), - data) - - def delete(self, endpoint, data = None): - return self._make_api_request('DELETE', - "/v%s/%s" % (self._api_version, endpoint), - data) - - def _make_api_request(self, method, endpoint, data = None): - # Convert data to string if it's not yet in this format - if data and not isinstance(data, str): - data = json.dumps(data).encode('utf-8') - - headers = { 'Content-type': 'application/json' } - if self._last_known_config: - headers['X-If-Configuration-Matches'] = self._last_known_config - - self._http_client.request(method, endpoint, data, - headers=headers) - - response = self._http_client.getresponse() - - status = response.status - body = response.read() - - # Make sure to use the X - if 'X-Configuration-Tag' in response.getheaders(): - self._last_known_config = response.getheaders()['X-Configuration-Tag'].decode('utf-8') - - print('Status:', status) - - # If you want verbose debugging - # print('Body:', body) - - print() - - result = json.loads(body.decode('utf-8')) - - if self._debug == True: - print(json.dumps(result, sort_keys=True, indent=4)) - - return result - - # Specific API requests - def get_version(self): - version = self.get('version') - return version['flocker'] - - def create_volume(self, name, size_in_gb, primary_id, profile = None): - if not isinstance(size_in_gb, int): - print('Error! Size must be an integer!') - exit(1) - - data = { - 'primary': primary_id, - 'maximum_size': size_in_gb << 30, - 'metadata': { - 'name': name - } - } - - if profile: - data['metadata']['clusterhq:flocker:profile'] = profile - - return self.post('configuration/datasets', data) - - def move_volume(self, volume_id, new_primary_id): - data = { 'primary': new_primary_id } - return self.post('configuration/datasets/%s' % volume_id, data) - - def delete_volume(self, dataset_id): - return self.delete('configuration/datasets/%s' % dataset_id) - - def get_volumes(self): - return self.get('configuration/datasets') - - def get_nodes(self): - return self.get('state/nodes') - - def get_leases(self): - return self.get('configuration/leases') - - def release_lease(self, dataset_id): - return self.delete('configuration/leases/%s' % dataset_id) - - def acquire_lease(self, dataset_id, node_id, expires = None): - data = { 'dataset_id': dataset_id, - 'node_uuid': node_id, - 'expires': expires } - return self.post('configuration/leases', data) - -if __name__ == '__main__': - api = FlockerApi(debug = True) - - # Show us the version of Flocker - print("Version:", api.get_version()) - - # Get current volumes (datasets) - print('Datasets:') - datasets = api.get_volumes() - - print('Nodes:') - nodes = api.get_nodes() - first_node = nodes[0]['uuid'] - - print('Trying to reuse the primary from returned list') - primary_id = datasets[0]['primary'] - print('Primary:', primary_id) - - print('Create volume:') - # Create a Flocker volume of size 10GB - dataset_create_result = api.create_volume('my-test-volume3', - 10, - primary_id, - profile = "gold") - volume_id = dataset_create_result['dataset_id'] - - print('Move volume (to the same node):') - api.move_volume(volume_id, primary_id) - - print('Acquire lease:') - api.acquire_lease(volume_id, first_node) - - print('Leases:') - leases = api.get_leases() - lease_id = leases[0] - - print('Release lease:') - api.release_lease(volume_id) - - print('Delete volume:') - api.delete_volume(volume_id) From 102f44e3ed6ba63a099484e22cd4a3421b82725c Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Fri, 19 Aug 2016 22:17:44 +0000 Subject: [PATCH 11/13] Added a dynamic REST API client to python examples This should do most of the things that we want through CLI params though we still need to add the things we infer from the env vars. --- python/flocker_api.py | 222 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100755 python/flocker_api.py diff --git a/python/flocker_api.py b/python/flocker_api.py new file mode 100755 index 0000000..e849c12 --- /dev/null +++ b/python/flocker_api.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 + +import http.client +import inspect +import json +import os +import ssl +import tempfile + +class FlockerApi(object): + DEFAULT_PLUGIN_DIR = os.environ.get('CERT_DIR', '/etc/flocker') + + def __init__(self, api_version = 1, debug = False): + control_service = os.environ.get("CONTROL_SERVICE", "localhost") + control_port = os.environ.get("CONTROL_PORT", 4523) + + self._api_version = api_version + self._debug = debug + self._last_known_config = None + + key_file = os.environ.get("KEY_FILE", "%s/plugin.key" % self.DEFAULT_PLUGIN_DIR) + cert_file = os.environ.get("CERT_FILE", "%s/plugin.crt" % self.DEFAULT_PLUGIN_DIR) + ca_file = os.environ.get("CA_FILE", "%s/cluster.crt" % self.DEFAULT_PLUGIN_DIR) + + # Create a certificate chain and then pass that into the SSL system. + cert_with_chain_tempfile = tempfile.NamedTemporaryFile() + + temp_cert_with_chain_path = cert_with_chain_tempfile.name + os.chmod(temp_cert_with_chain_path, 0o0600) + + # Write our cert and append the CA cert to build the chain + with open(cert_file, 'rb') as cert_file_obj: + cert_with_chain_tempfile.write(cert_file_obj.read()) + + cert_with_chain_tempfile.write('\n'.encode('utf-8')) + + with open(ca_file, 'rb') as cacert_file_obj: + cert_with_chain_tempfile.write(cacert_file_obj.read()) + + # Reset file pointer for the SSL context to read it properly + cert_with_chain_tempfile.seek(0) + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ssl_context.load_cert_chain(temp_cert_with_chain_path, key_file) + + self._http_client = http.client.HTTPSConnection(control_service, + control_port, + context=ssl_context) + + # XXX: These should really be generic functions created dynamically + def get(self, endpoint, data = None): + return self._make_api_request('GET', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def post(self, endpoint, data = None): + return self._make_api_request('POST', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def delete(self, endpoint, data = None): + return self._make_api_request('DELETE', + "/v%s/%s" % (self._api_version, endpoint), + data) + + def _make_api_request(self, method, endpoint, data = None): + # Convert data to string if it's not yet in this format + if data and not isinstance(data, str): + data = json.dumps(data).encode('utf-8') + + headers = { 'Content-type': 'application/json' } + if self._last_known_config: + headers['X-If-Configuration-Matches'] = self._last_known_config + + self._http_client.request(method, endpoint, data, + headers=headers) + + response = self._http_client.getresponse() + + status = response.status + body = response.read() + + # Make sure to use the X + if 'X-Configuration-Tag' in response.getheaders(): + self._last_known_config = response.getheaders()['X-Configuration-Tag'].decode('utf-8') + + print('Status:', status) + + # If you want verbose debugging + # print('Body:', body) + + print() + + result = json.loads(body.decode('utf-8')) + + if self._debug == True: + print(json.dumps(result, sort_keys=True, indent=4)) + + return result + + # XXX: Dummy decorator that allows us to indicate what methods are part + # of the CLI + def cli_method(func): + return func + + # XXX: Yeah it's gnarly :( + # TODO: Get a better way to introspect that's not a static array + def get_methods(self): + source = inspect.getsourcelines(FlockerApi)[0] + for index, line in enumerate(source): + line = line.strip() + if line.strip() == '@cli_method': + nextLine = source[ index + 1 ] + name = nextLine.split('def')[1].split('(')[0].strip() + yield(name) + + # Specific API requests + @cli_method + def get_version(self): + """ + Gets version of the Flocker service + """ + version = self.get('version') + return version['flocker'] + + @cli_method + def create_volume(self, name, size_in_gb, primary_id, profile = None): + if not isinstance(size_in_gb, int): + print('Error! Size must be an integer!') + exit(1) + + data = { + 'primary': primary_id, + 'maximum_size': size_in_gb << 30, + 'metadata': { + 'name': name + } + } + + if profile: + data['metadata']['clusterhq:flocker:profile'] = profile + + return self.post('configuration/datasets', data) + + + @cli_method + def move_volume(self, volume_id, new_primary_id): + data = { 'primary': new_primary_id } + return self.post('configuration/datasets/%s' % volume_id, data) + + @cli_method + def delete_volume(self, dataset_id): + return self.delete('configuration/datasets/%s' % dataset_id) + + @cli_method + def get_volumes(self): + return self.get('configuration/datasets') + + @cli_method + def get_nodes(self): + return self.get('state/nodes') + + @cli_method + def get_leases(self): + return self.get('configuration/leases') + + @cli_method + def release_lease(self, dataset_id): + return self.delete('configuration/leases/%s' % dataset_id) + + @cli_method + def acquire_lease(self, dataset_id, node_id, expires = None): + data = { 'dataset_id': dataset_id, + 'node_uuid': node_id, + 'expires': expires } + return self.post('configuration/leases', data) + +if __name__ == '__main__': + # We only parse args if we're invoked as a script + import argparse + api = FlockerApi(debug = True) + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest='action') + + # Dynamically add all relevant cli methods + for method_name in api.get_methods(): + func = getattr(api, method_name) + help_doc = func.__doc__ or "No documentation" + help_line = '%s. See "%s -help" for more options' % (help_doc, method_name) + args = inspect.getargspec(func) + + parser_for_method = subparsers.add_parser(method_name, help = help_line) + # Mandatory args + for index, arg in enumerate(args.args): + # Skip 'self' + if index == 0: + continue + + # Divide into things with defaults and things without + if index < len(args.args) - len(args.defaults or []): + parser_for_method.add_argument(arg) + else: + parser_for_method.add_argument('--%s' % arg, default=args.defaults[len(args.args) - index - 1]) + + parsed_args = parser.parse_args() + + action = parsed_args.action + + print("Action:", parsed_args.action) + + func = getattr(api, action) + args = inspect.getargspec(func) + args_to_send = [] + for index, arg in enumerate(args.args): + # Skip 'self' + if index == 0: + continue + + args_to_send.append(vars(parsed_args)[arg]) + print('Args:', args_to_send) + func(*args_to_send) From 1b3f28053dfeb132316d16f9f58621f3adc22f48 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Sun, 21 Aug 2016 15:20:14 +0000 Subject: [PATCH 12/13] Made get_* methods be list_* in flocker_api script This should be a bit more intuitive. --- python/flocker_api.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/flocker_api.py b/python/flocker_api.py index e849c12..6537d71 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -126,8 +126,7 @@ def get_version(self): @cli_method def create_volume(self, name, size_in_gb, primary_id, profile = None): if not isinstance(size_in_gb, int): - print('Error! Size must be an integer!') - exit(1) + size_in_gb = int(size_in_gb) data = { 'primary': primary_id, @@ -153,15 +152,15 @@ def delete_volume(self, dataset_id): return self.delete('configuration/datasets/%s' % dataset_id) @cli_method - def get_volumes(self): + def list_volumes(self): return self.get('configuration/datasets') @cli_method - def get_nodes(self): + def list_nodes(self): return self.get('state/nodes') @cli_method - def get_leases(self): + def list_leases(self): return self.get('configuration/leases') @cli_method From dc571108e36adea76c13517ccbbfe3c8654b4a24 Mon Sep 17 00:00:00 2001 From: Srdjan Grubor Date: Sun, 21 Aug 2016 15:50:21 +0000 Subject: [PATCH 13/13] Added more help documentation to flocker_api.py Merged changes from: https://github.com/sgnn7/flocker-rest-client/blob/master/flocker_api.py --- python/flocker_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/flocker_api.py b/python/flocker_api.py index 6537d71..60dda70 100755 --- a/python/flocker_api.py +++ b/python/flocker_api.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# +# Description: Flexible Flocker REST API Client +# License: LGPLv2 +# Maintainer: Srdjan Grubor import http.client import inspect