From 59c72633001fa5ca5b12ff0f3a3db57b41ed7e48 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 13 Feb 2019 00:04:53 -0500 Subject: [PATCH 01/10] initial import of remote cluster support --- python/phylanx/ast/clusters.py | 28 ++++++++++++++++++++++++++++ python/phylanx/ast/transducer.py | 17 +++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 python/phylanx/ast/clusters.py diff --git a/python/phylanx/ast/clusters.py b/python/phylanx/ast/clusters.py new file mode 100644 index 000000000..5b1d3ee6f --- /dev/null +++ b/python/phylanx/ast/clusters.py @@ -0,0 +1,28 @@ +# Copyright (c) 2018 Christopher Taylor +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +import requests + +class cluster(object): + def __init_(self, args): + self.args = args + + def send(self): + raise NotImplementedError("cluster::send not impl error.") + +class k8cluster(cluster): + def __init__(self, args): + cluster.__init__(self, args) + + @overrides(cluster) + def sends(self): + resp = requests.post(url=self.args['url'], args=self.args) + data = resp.json() + +def create_cluster(args): + cluster_type = args['type'] + if cluster_type == 'k8' or cluster_type == 'kubernetes': + if 'config' not in args: + raise NotImplementedError("k8cluster config missing.") + return k8cluster(args['config']) diff --git a/python/phylanx/ast/transducer.py b/python/phylanx/ast/transducer.py index 2d8dd5a61..fa07d4916 100644 --- a/python/phylanx/ast/transducer.py +++ b/python/phylanx/ast/transducer.py @@ -14,7 +14,7 @@ from phylanx import execution_tree from phylanx.ast import generate_ast as generate_phylanx_ast from phylanx.exceptions import InvalidDecoratorArgumentError - +from phylanx.clusters import create_cluster def Phylanx(__phylanx_arg=None, **kwargs): class __PhylanxDecorator(object): @@ -22,7 +22,7 @@ def __init__(self, f): """ :function:f the decorated funtion. """ - valid_kwargs = ['debug', 'target', 'compiler_state', 'performance'] + valid_kwargs = ['debug', 'target', 'compiler_state', 'performance', 'cluster'] self.backends_map = {'PhySL': PhySL, 'OpenSCoP': OpenSCoP} self.backend = self.get_backend(kwargs.get('target')) @@ -36,6 +36,14 @@ def __init__(self, f): else: kwargs['fglobals'] = f.__globals__ + # determine if code is sent to a remote cluster + # + self.remote = None + if 'cluster' in kwargs.keys(): + self.remote = create_cluster(kwargs['cluster']) + else: + raise NotImplementedError("Unknown Phylanx argument '%s'" % (kwargs['cluster'],)) + python_src = self.get_python_src(f) python_ast = self.get_python_ast(python_src, f) @@ -79,10 +87,11 @@ def __call__(self, *args): raise NotImplementedError( "OpenSCoP kernels are not yet callable.") - result = self.backend.call(args) + if self.remote is not None or self.remote != None: + return self.remote.send(self.__src__) + result = self.backend.call(args) self.__perfdata__ = self.backend.__perfdata__ - return result def generate_ast(self): From 0ed45b9bb832e1fb1e13b3f291e60d5fd97eeb05 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 13 Feb 2019 00:23:18 -0500 Subject: [PATCH 02/10] initial import of remote k8 cluster support --- python/phylanx/ast/clusters.py | 41 ++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/python/phylanx/ast/clusters.py b/python/phylanx/ast/clusters.py index 5b1d3ee6f..d49fc23da 100644 --- a/python/phylanx/ast/clusters.py +++ b/python/phylanx/ast/clusters.py @@ -2,27 +2,54 @@ # # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +''' +see this documentation: + +https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/#accessing-the-cluster-api +https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ +https://github.com/kubernetes-client/python/blob/master/examples/exec.py +''' + import requests +from base64 import b64encode class cluster(object): def __init_(self, args): self.args = args def send(self): - raise NotImplementedError("cluster::send not impl error.") + raise NotImplementedError("cluster::send not implemented.") class k8cluster(cluster): def __init__(self, args): cluster.__init__(self, args) @overrides(cluster) - def sends(self): - resp = requests.post(url=self.args['url'], args=self.args) - data = resp.json() + def sends(self, physl_ir_str): + if 'jobconfig' not in self.args: + raise Error("k8cluster jobconfig not defined.") + if 'url' not in self.args: + raise Error("k8cluster url not defined.") + + env_ir = [ {'name' : 'PHYSL_IR', 'value' : b64encode(physl_ir_str) } ] + + if 'env' not in self.args['jobconfig']['spec']['containers']: + self.args['jobconfig']['spec']['containers']['env'] = list() + + self.args['jobconfig']['spec']['containers']['env'].append(env_ir) + + resp = requests.post(url=self.args['url'], args=self.args['jobconfig']) + return resp.json() def create_cluster(args): + if 'type' not in args: + raise Error("create_cluster type not defined.") + cluster_type = args['type'] + + if 'jobconfig' not in args: + raise Error("create_cluster jobconfig not defined.") + if cluster_type == 'k8' or cluster_type == 'kubernetes': - if 'config' not in args: - raise NotImplementedError("k8cluster config missing.") - return k8cluster(args['config']) + return k8cluster(args['jobconfig']) From 90474bf23de58c31a5c948dea3d397ed9232bc3a Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 4 May 2019 10:24:51 -0400 Subject: [PATCH 03/10] added deploy --- python/phylanx/ast/clusters.py | 55 ---------------------------------- python/phylanx/ast/physl.py | 1 + 2 files changed, 1 insertion(+), 55 deletions(-) delete mode 100644 python/phylanx/ast/clusters.py diff --git a/python/phylanx/ast/clusters.py b/python/phylanx/ast/clusters.py deleted file mode 100644 index d49fc23da..000000000 --- a/python/phylanx/ast/clusters.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2018 Christopher Taylor -# -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -''' -see this documentation: - -https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/#accessing-the-cluster-api -https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ -https://github.com/kubernetes-client/python/blob/master/examples/exec.py -''' - -import requests -from base64 import b64encode - -class cluster(object): - def __init_(self, args): - self.args = args - - def send(self): - raise NotImplementedError("cluster::send not implemented.") - -class k8cluster(cluster): - def __init__(self, args): - cluster.__init__(self, args) - - @overrides(cluster) - def sends(self, physl_ir_str): - if 'jobconfig' not in self.args: - raise Error("k8cluster jobconfig not defined.") - if 'url' not in self.args: - raise Error("k8cluster url not defined.") - - env_ir = [ {'name' : 'PHYSL_IR', 'value' : b64encode(physl_ir_str) } ] - - if 'env' not in self.args['jobconfig']['spec']['containers']: - self.args['jobconfig']['spec']['containers']['env'] = list() - - self.args['jobconfig']['spec']['containers']['env'].append(env_ir) - - resp = requests.post(url=self.args['url'], args=self.args['jobconfig']) - return resp.json() - -def create_cluster(args): - if 'type' not in args: - raise Error("create_cluster type not defined.") - - cluster_type = args['type'] - - if 'jobconfig' not in args: - raise Error("create_cluster jobconfig not defined.") - - if cluster_type == 'k8' or cluster_type == 'kubernetes': - return k8cluster(args['jobconfig']) diff --git a/python/phylanx/ast/physl.py b/python/phylanx/ast/physl.py index ae83a568b..c863a9b49 100644 --- a/python/phylanx/ast/physl.py +++ b/python/phylanx/ast/physl.py @@ -11,6 +11,7 @@ import numpy as np import phylanx.execution_tree from phylanx import PhylanxSession +from deploy import * def physl_zip(loop): From 0fd5b7473b0e4eec0fd83b774944b67a8ee9a102 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 4 May 2019 10:26:36 -0400 Subject: [PATCH 04/10] more fixes --- python/phylanx/ast/transducer.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/phylanx/ast/transducer.py b/python/phylanx/ast/transducer.py index 93dc1ff1c..e6d851955 100644 --- a/python/phylanx/ast/transducer.py +++ b/python/phylanx/ast/transducer.py @@ -55,14 +55,6 @@ def __init__(self, f): else: kwargs['fglobals'] = f.__globals__ - # determine if code is sent to a remote cluster - # - self.remote = None - if 'cluster' in kwargs.keys(): - self.remote = create_cluster(kwargs['cluster']) - else: - raise NotImplementedError("Unknown Phylanx argument '%s'" % (kwargs['cluster'],)) - python_src = self.get_python_src(f) python_ast = self.get_python_ast(python_src, f) @@ -151,7 +143,7 @@ def __call__(self, *args): "OpenSCoP kernels are not yet callable.") result = self.backend.call(map(self.map_decorated, args)) - result = self.backend.call(args) + self.__perfdata__ = self.backend.__perfdata__ return result From 2c1f434ae9b3306048117d58b67a61ed465b1b5c Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 4 May 2019 10:28:20 -0400 Subject: [PATCH 05/10] added full module path --- python/phylanx/ast/physl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/phylanx/ast/physl.py b/python/phylanx/ast/physl.py index c863a9b49..3b3a1023a 100644 --- a/python/phylanx/ast/physl.py +++ b/python/phylanx/ast/physl.py @@ -11,7 +11,7 @@ import numpy as np import phylanx.execution_tree from phylanx import PhylanxSession -from deploy import * +from phylanx.ast.deploy import * def physl_zip(loop): From 2069e3d295042ee8c7094aa3b59891546915df27 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 4 May 2019 10:29:21 -0400 Subject: [PATCH 06/10] rm lame code module --- python/phylanx/ast/transducer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/phylanx/ast/transducer.py b/python/phylanx/ast/transducer.py index e6d851955..bbc0aeae6 100644 --- a/python/phylanx/ast/transducer.py +++ b/python/phylanx/ast/transducer.py @@ -15,7 +15,6 @@ from phylanx import execution_tree from phylanx.ast import generate_ast as generate_phylanx_ast from phylanx.exceptions import InvalidDecoratorArgumentError -from phylanx.clusters import create_cluster class LambdaExtractor(ast.NodeVisitor): _ast = None From 216a4f17d072ee97307bd950a547e0f087258521 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 4 May 2019 10:30:25 -0400 Subject: [PATCH 07/10] added return values --- python/phylanx/ast/deploy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/phylanx/ast/deploy.py b/python/phylanx/ast/deploy.py index b813d182d..6369a4c0a 100644 --- a/python/phylanx/ast/deploy.py +++ b/python/phylanx/ast/deploy.py @@ -57,5 +57,7 @@ def __call__(self): except ApiException as e: if e.status != 404: print('unknown error: %s' % (str(e),)) + return None self.set_result(resp) + return resp From 23754cdaa8df93bc9894318f7ca51dae641293ac Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 5 May 2019 14:20:02 -0400 Subject: [PATCH 08/10] updated with boost license --- python/phylanx/ast/deploy.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/phylanx/ast/deploy.py b/python/phylanx/ast/deploy.py index 6369a4c0a..970db938a 100644 --- a/python/phylanx/ast/deploy.py +++ b/python/phylanx/ast/deploy.py @@ -1,8 +1,14 @@ +# Copyright (c) 2019 Christopher Taylor +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +# uses this library: https://github.com/kubernetes-client/python +# from kubernetes import client, config, utils from kubernetes.client.rest import ApiException -# uses this library -# https://github.com/litl/backoff +# uses this library: https://github.com/litl/backoff # import backoff From 413afba5689a4d620a8ee5c35ed20f4eabf4417d Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 5 May 2019 14:20:58 -0400 Subject: [PATCH 09/10] fix --- python/phylanx/ast/physl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/phylanx/ast/physl.py b/python/phylanx/ast/physl.py index 3b3a1023a..cf67284bb 100644 --- a/python/phylanx/ast/physl.py +++ b/python/phylanx/ast/physl.py @@ -11,7 +11,7 @@ import numpy as np import phylanx.execution_tree from phylanx import PhylanxSession -from phylanx.ast.deploy import * +from phylanx.ast.deploy import deployer def physl_zip(loop): From 0914eec2a6880f63dc46b08b2262c1065d5ea607 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 7 May 2019 00:27:30 -0400 Subject: [PATCH 10/10] fixed up things, added first pass at result collection --- python/phylanx/ast/deploy.py | 83 ++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/python/phylanx/ast/deploy.py b/python/phylanx/ast/deploy.py index 970db938a..0a7285a65 100644 --- a/python/phylanx/ast/deploy.py +++ b/python/phylanx/ast/deploy.py @@ -1,16 +1,19 @@ -# Copyright (c) 2019 Christopher Taylor +# Copyright (c) 2019 Christopher Taylor # # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) # uses this library: https://github.com/kubernetes-client/python # -from kubernetes import client, config, utils +import kubernetes.client +from kubernetes.client import Configuration +from kubernetes.client.apis import core_v1_api from kubernetes.client.rest import ApiException # uses this library: https://github.com/litl/backoff # -import backoff +import backoff + class deployer(object): ''' @@ -19,51 +22,87 @@ class deployer(object): ''' def __init__(self, physl_str): self.physl = physl_str - self.result = None def __call__(self): pass - def set_result(self, result): - self.result = result - def result(self): - return self.result +class deployedresult(object): + ''' + data structure to manage posting physl code + deployments + ''' + def __init__(self): + pass + + def get(self): + pass + + +class kubernetes_deployer_result(deployedresult): + ''' + data structure to manage getting result of + physl code deployed to a kubernetes cluster + ''' + def __init__(self, kubernetes_pod_deploy_response, + kubernetes_config, log_path): + deployedresult.__init__(self) + self.kpod_response = kubernetes_pod_deploy_response + self.kpod_config = kubernetes_config + self.success = False + self.log_path = log_path + + if self.kpod_response is not None: + self.success = True + + def get(self): + api_instance = kubernetes.client.LogsApi( + kubernetes.client.ApiClient(self.kpod_config)) + + thread = None + try: + thread = api_instance.log_file_handler_with_http_info(self.logpath) + except ApiException as e: + print("Exception when calling LogsApi->log_file_handler: %s\n" % e) + + if thread is not None: + return thread.get() + + return None + class kubernetes_deployer(deployer): ''' data structure to manage posting physl code to a kubernetes cluster ''' - def __init__(self, physl_str, pod_manifest, base=10, cap=300): + def __init__(self, physl_str, pod_manifest, + kubernetes_config, base=10, cap=300): deployer.__init__(self, physl_str) self.manifest = pod_manifest + self.kpod_config = kubernetes_config self.name = pod_manifest['metadata']['name'] - self.base = 10 + self.base = base self.cap = cap - @backoff.on_predicate(backoff.expo, lambda x: x.status.phase != 'Pending'): - def post_loop(self): + @backoff.on_predicate(backoff.expo, lambda x: x.status.phase != 'Pending') + def post_pod_loop(self): ''' posts using exponential backoff ''' - return api.read_namespaced_pod(name=self.name, namespace='default') + return self.api.read_namespaced_pod( + name=self.name, namespace='default') - @override def __call__(self): - config.load_kube_config() - c = Configuration() - c.assert_hostname = False - Configuration.set_default(c) - api = core_v1_api.CoreV1Api() + Configuration.set_default(self.kpod_config) + self.api = core_v1_api.CoreV1Api() resp = None try: - resp = self.post_loop() + resp = self.post_pod_loop() except ApiException as e: if e.status != 404: print('unknown error: %s' % (str(e),)) return None - self.set_result(resp) - return resp + return kubernetes_deployer_result(resp)