Merge branch 'dev' of github.com:WeBankPartners/wecube-plugins-terraf…

…orm into dev

apps/api/apibase_backend.py

@@ -44,6 +44,30 @@ def fetech_property(instance_define, define_columns):
     return res
 
 
+def fetch_columns(datas, columns):
+    if len(columns) == 0:
+        c_data = []
+        for data in datas:
+            if isinstance(data, list):
+                c_data += data
+            else:
+                c_data.append(data)
+        return c_data
+    column = columns.pop(0)
+    if isinstance(datas, list):
+        x_data = []
+        for data in datas:
+            x_data.append(data.get(column))
+        datas = x_data
+        return fetch_columns(datas, columns)
+    elif isinstance(datas, dict):
+        datas = datas.get(column)
+        return fetch_columns(datas, columns)
+    else:
+        logger.info("data is not dict/list, no columns %s filter, skip.." % column)
+        return datas
+
+
 class ApiBackendBase(TerraformResource):
     def __init__(self):
         super(ApiBackendBase, self).__init__()
@@ -328,7 +352,8 @@ def source_run_import(self, rid, provider, region, label_name,
                 # if workpath:
                 #     self.rollback_workspace(workpath)
                 logger.info(traceback.format_exc())
-                logger.info("resource import failed. continue ... ")
+                # logger.info("resource import failed. continue ... ")
+                raise e
 
         logger.info("state file not recovery, continue apply resource ... ")
         return True
@@ -556,17 +581,19 @@ def read_query_result_controller(self, provider, result, data_source_argument):
 
         logger.info(format_json_dumps(result))
         try:
+            instance_define = []
             _data = result.get("resources")[0]
             _instances = _data.get("instances")[0]
             _attributes = _instances.get("attributes")
 
             outlines = data_source_argument.split(".")
-            for outline in outlines:
-                if outline:
-                    _attributes = _attributes.get(outline)
-
-            instance_list = _attributes
-            instance_define = instance_list
+            # for outline in outlines:
+            #     if outline:
+            #         _attributes = _attributes.get(outline)
+            #
+            # instance_list = _attributes
+            # instance_define = instance_list
+            instance_define = fetch_columns(datas=_attributes, columns=outlines)
         except:
             logger.info(traceback.format_exc())
             raise ValueError("query remote source failed, result read faild")
@@ -655,6 +682,7 @@ def run_query(self, rid, region, zone,
                                                         data=out_data)
 
             x_json = self.reverse_asset_object(provider=provider_object["name"], data=x_json)
+            # x_json.update(query_data)
 
             if x_json.get("instance_type"):
                 instance_type, resource_info = InstanceTypeObject().convert_asset(provider=provider_object["name"],

apps/api/network/eip.py

@@ -46,7 +46,7 @@ def __init__(self):
         self.resource_keys_config = None
 
 
-class EipBackendApi(Common, ApiBase):
+class EipBackendApi(Common, ApiBackendBase):
     def __init__(self):
         super(EipBackendApi, self).__init__()
         self.resource_name = "eip"

apps/api/network/nat_gateway.py

@@ -52,7 +52,8 @@ def generate_create_data(self, zone, create_data, **kwargs):
                          "subnet_id": create_data.get("subnet_id")}
 
         create_data = {"eip": create_data.get("eip"),
-                       "name": create_data.get("name")}
+                       "name": create_data.get("name"),
+                       "bandwidth": create_data.get("bandwidth")}
 
         return create_data, r_create_data
 

apps/api/network/route_entry.py

@@ -6,9 +6,9 @@
 from lib.logs import logger
 from lib.json_helper import format_json_dumps
 from apps.common.convert_keys import define_relations_key
-from apps.api.apibase import ApiBase
-from apps.api.configer.provider import ProviderApi
 from apps.background.resource.resource_base import CrsObject
+from apps.api.apibase import ApiBase
+from apps.api.conductor.valueReverse import ValueResetConductor
 from apps.api.apibase_backend import ApiBackendBase
 
 
@@ -25,29 +25,33 @@ def before_keys_checks(self, provider, create_data, is_update=None):
 
         vpc_id = create_data.get("vpc_id")
         route_table_id = create_data.get("route_table_id")
+        next_hub = create_data.get("next_hub")
 
         self.resource_info(provider)
         resource_property = self.resource_keys_config["resource_property"]
         _vpc_status = define_relations_key("vpc_id", vpc_id, resource_property.get("vpc_id"))
         _rt_status = define_relations_key("route_table_id", route_table_id, resource_property.get("route_table_id"))
+        _ne_status = define_relations_key("next_hub", next_hub, resource_property.get("next_hub"))
 
         ext_info = {}
         if vpc_id and (not _vpc_status):
             ext_info["vpc_id"] = CrsObject("vpc").object_resource_id(vpc_id)
         if route_table_id and (not _rt_status):
             ext_info["route_table_id"] = CrsObject(self.owner_resource).object_resource_id(route_table_id)
+        if next_hub and (not _ne_status):
+            ext_info["next_hub"] = CrsObject(self.owner_resource).object_resource_assetid(next_hub)
 
         logger.info("before_keys_checks add info: %s" % (format_json_dumps(ext_info)))
         return ext_info
 
     def generate_create_data(self, zone, create_data, **kwargs):
         r_create_data = {"vpc_id": create_data.get("vpc_id"),
+                         "next_hub": create_data.get("next_hub"),
                          "route_table_id": create_data.get("route_table_id")}
 
         create_data = {"name": create_data.get("name"),
                        "destination": create_data.get("destination"),
                        "next_type": create_data.get("next_type"),
-                       "next_hub": create_data.get("next_hub")
                        }
 
         return create_data, r_create_data
@@ -77,12 +81,62 @@ def __init__(self):
         self.resource_keys_config = None
 
     def before_source_asset(self, provider, query_data):
-        for key in ["route_table_id", "vpc_id"]:
+        for key in ["route_table_id", "vpc_id", "next_hub"]:
             if query_data.get(key):
                 query_data[key] = CrsObject().object_asset_id(query_data.get(key))
 
         return query_data
 
     def reverse_asset_ids(self):
-        return ['vpc_id', "route_table_id"]
+        return ['vpc_id', "route_table_id", "next_hub"]
+
+    def run_query(self, rid, region, zone,
+                  provider_object, provider_info,
+                  query_data, **kwargs):
+        '''
+
+        :param rid:
+        :param region:
+        :param zone:
+        :param owner_id:
+        :param relation_id:
+        :param create_data:
+        :param extend_info:
+        :param kwargs:
+        :return:
+        '''
+
+        # extend_info = extend_info or {}
+        label_name = self.resource_name + "_q_" + rid
+
+        define_json, resource_keys_config = self.source_filter_controller(provider_name=provider_object["name"],
+                                                                          label_name=label_name,
+                                                                          query_data=query_data
+                                                                          )
+
+        result = self._run_create_and_read_result(rid, provider=provider_object["name"],
+                                                  region=region, provider_info=provider_info,
+                                                  define_json=define_json)
+
+        data_source_argument = resource_keys_config.get("data_source_argument")
+        output_json = self.read_query_result_controller(provider_object["name"], result,
+                                                        data_source_argument)
+
+        result_list = []
+        for out_data in output_json:
+            count = 0
+            for _, x_value in out_data.items():
+                if not x_value:
+                    count += 1
+            if count == len(out_data):
+                logger.info("out data columns is null, skip ...")
+                continue
+
+            x_json = ValueResetConductor().reset_values(provider=provider_object["name"],
+                                                        resource_name=self.resource_name,
+                                                        data=out_data)
+
+            x_json = self.reverse_asset_object(provider=provider_object["name"], data=x_json)
+            result_list.append(x_json)
 
+        return result_list

apps/background/resource/resource_base.py

@@ -184,6 +184,12 @@ def object_resource_id(self, rid):
             raise local_exceptions.ValueValidateError(self.resource_name, "资源 %s 不存在" % rid)
         return data["resource_id"]
 
+    def object_resource_assetid(self, rid):
+        data = self.show(rid)
+        if not data:
+            return rid
+        return data["resource_id"]
+
     def object_asset_id(self, rid):
         data = self.show(rid)
         if data:

apps/common/convert_keys.py

@@ -8,6 +8,7 @@
 
 
 def validate_convert_key(defines):
+    x_res = {}
     for key, define in defines.items():
         if (not isinstance(define, (basestring, dict))) or isinstance(define, list):
             raise ValueError("错误的定义 合法值为 string "
@@ -18,8 +19,14 @@ def validate_convert_key(defines):
         if isinstance(define, dict):
             if define.get("type", "string") not in ["string", "json", "int", "float", "list", "bool"]:
                 raise ValueError("未知的类型约束 %s" % define.get("type"))
-            if define.get("allow_null", 1) not in [0, 1]:
+            if define.get("allow_null", 1) not in [0, 1, '0', '1']:
                 raise ValueError("allow_null 合法值为 0/1")
+            else:
+                define["allow_null"] = int(define.get("allow_null", 1))
+
+        x_res[key] = define
+
+    return x_res
 
 
 def validate_convert_value(defines):
@@ -62,7 +69,10 @@ def validate_type(value, type):
         try:
             if isinstance(value, basestring):
                 if value.startswith("["):
-                    value = json.loads(value)
+                    try:
+                        value = json.loads(value)
+                    except:
+                        value = eval(value)
                 elif "," in value:
                     value = value.split(",")
                 elif ";" in value:

apps/controller/configer/defines.py

@@ -226,7 +226,7 @@
             "method": "POST",
             "notnull": ['name', 'provider', 'vpc_id', 'region'],
             "inputParameters": ['id', 'name', 'secret', 'provider', 'vpc_id', 'subnet_id', 'eip', 'zone', 'region',
-                                'asset_id', 'resource_id', 'extend_info'],
+                                'asset_id', 'resource_id', 'extend_info', 'bandwidth'],
             "outputParameters": ['errorMessage', 'errorCode', 'ipaddress', 'id', 'resource_id']
         },
         "destroy": {
@@ -242,7 +242,7 @@
             "notnull": ["region", "provider"],
             "inputParameters": ['region', 'secret', 'provider', 'resource_id', 'name', 'type', 'tag', 'vpc_id'],
             "outputParameters": ['region', 'secret', 'provider', 'resource_id', 'errorMessage', 'errorCode', 'name',
-                                 'vpc_id', 'eip', 'ipaddress']
+                                 'vpc_id', 'eip', 'ipaddress', 'bandwidth']
         }
     },
     "eip": {
@@ -365,7 +365,8 @@
             "path": "/terraform/v1/storage/backend/disk/source",
             "method": "POST",
             "notnull": ["region", "provider"],
-            "inputParameters": ['region', 'secret', 'provider', 'resource_id', 'name', 'instance_id', 'type', 'tag'],
+            "inputParameters": ['region', 'secret', 'provider', 'resource_id',
+                                'name', 'instance_id', 'type', 'tag', 'zone'],
             "outputParameters": ['region', 'secret', 'provider', 'resource_id', 'errorMessage', 'errorCode', 'name',
                                  'type', 'size', 'zone', "charge_type"]
         }

apps/controller/configer/model_args.py

@@ -46,7 +46,8 @@
         "name",
         "vpc_id",
         "subnet_id",
-        "eip"
+        "eip",
+        "bandwidth"
     ],
     "peer_connection": [
         "name",
@@ -516,7 +517,8 @@
         "name",
         "type",
         "tag",
-        "vpc_id"
+        "vpc_id",
+        "bandwidth"
     ],
     "peer_connection": [
         "resource_id",
@@ -558,7 +560,8 @@
         "name",
         "instance_id",
         "type",
-        "tag"
+        "tag",
+        "zone"
     ],
     "disk_attach": [
         "resource_id",

apps/controller/configer/provider_controller.py

@@ -66,7 +66,7 @@ def create(self, request, data, **kwargs):
         name = data.get("name")
         extend_info = validation.validate_dict("extend_info", data.get("extend_info")) or {}
         provider_property = validation.validate_dict("provider_property", data.get("provider_property")) or {}
-        validate_convert_key(provider_property)
+        provider_property = validate_convert_key(provider_property)
         validate_convert_value(extend_info)
         property_necessary(resource_name="provider",
                            resource_property=provider_property)
@@ -115,7 +115,7 @@ def update(self, request, data, **kwargs):
 
         if data.get("provider_property") is not None:
             provider_property = validation.validate_dict("provider_property", data.get("provider_property")) or {}
-            validate_convert_key(provider_property)
+            provider_property = validate_convert_key(provider_property)
             property_necessary(resource_name="provider",
                                resource_property=provider_property)
             data["provider_property"] = json.dumps(provider_property)

apps/controller/configer/resource_controller.py

@@ -87,10 +87,10 @@ def create(self, request, data, **kwargs):
         #     if not isinstance(value, basestring):
         #         raise ValueError("data_source_output 为key-value定义")
 
-        validate_convert_key(resource_property)
+        resource_property = validate_convert_key(resource_property)
         validate_convert_value(extend_info)
         validate_convert_value(resource_output)
-        validate_convert_key(data_source_output)
+        data_source_output = validate_convert_key(data_source_output)
         property_necessary(resource_name=data["resource_name"],
                            resource_property=resource_property)
 
@@ -158,7 +158,7 @@ def update(self, request, data, **kwargs):
 
         if data.get("resource_property") is not None:
             resource_property = validation.validate_dict("resource_property", data.get("resource_property")) or {}
-            validate_convert_key(resource_property)
+            resource_property = validate_convert_key(resource_property)
             property_necessary(resource_name=data["resource_name"],
                                resource_property=resource_property)
 
@@ -181,6 +181,7 @@ def update(self, request, data, **kwargs):
 
         if data.get("data_source_output") is not None:
             data_source_output = validation.validate_dict("data_source_output", data.get("data_source_output"))
+            data_source_output = validate_convert_key(data_source_output)
 
             property_necessary(resource_name=data["resource_name"],
                                resource_property=data_source_output)

apps/controller/network/nat_controller.py

@@ -43,14 +43,18 @@ def create(cls, resource, data, **kwargs):
         vpc_id = data.pop("vpc_id", None)
         subnet_id = data.pop("subnet_id", None)
         eip = data.pop("eip", None)
+        # eip = validation.validate_list("eip", data.pop("eip", None))
 
         asset_id = data.pop("asset_id", None)
         resource_id = data.pop("resource_id", None)
+        bandwidth = data.pop("bandwidth", None)
 
         extend_info = validation.validate_dict("extend_info", data.pop("extend_info", None))
         data.update(extend_info)
 
-        create_data = {"name": name, "vpc_id": vpc_id, "subnet_id": subnet_id, "eip": eip}
+        create_data = {"name": name, "vpc_id": vpc_id,
+                       "subnet_id": subnet_id, "eip": eip,
+                       "bandwidth": bandwidth}
         _, result = resource.create(rid=rid, provider=provider,
                                     region=region, zone=zone,
                                     secret=secret,