Merge pull request #26 from WeBankPartners/dev

Dev

Dockerfile

@@ -1,4 +1,4 @@
-FROM ccr.ccs.tencentyun.com/webankpartners/terrafrom-base:v1.0.3
+FROM ccr.ccs.tencentyun.com/webankpartners/terrafrom-base:v1.0.4
 
 ENV BASE_HOME=/app/terraform
 

build/default.json

@@ -37,5 +37,6 @@
   "terraform_file_path": "/data/terraform/",
   "terraform_cmd_path": "/usr/local/bin/terraform",
   "terraform_provider_os_arch": "linux_amd64",
-  "version": "{{PLUGIN_VERSION}}"
+  "version": "{{PLUGIN_VERSION}}",
+  "request_consumer_count": 6
 }

terraform-server/api/api.go

@@ -71,6 +71,8 @@ func init() {
 		&handlerFuncObj{Url: "/providers", Method: "GET", HandlerFunc: provider.ProviderList},
 		&handlerFuncObj{Url: "/providers", Method: "DELETE", HandlerFunc: provider.ProviderBatchDelete, LogOperation: true},
 		&handlerFuncObj{Url: "/providers", Method: "PUT", HandlerFunc: provider.ProviderBatchUpdate, LogOperation: true},
+		&handlerFuncObj{Url: "/providers/download", Method: "POST", HandlerFunc: provider.ProviderDownload, LogOperation: false},
+		&handlerFuncObj{Url: "/providers/upload", Method: "POST", HandlerFunc: provider.ProviderUpload, LogOperation: false},
 
 		&handlerFuncObj{Url: "/provider_infos", Method: "POST", HandlerFunc: provider.ProviderInfoBatchCreate, LogOperation: true},
 		&handlerFuncObj{Url: "/provider_infos", Method: "GET", HandlerFunc: provider.ProviderInfoList},

terraform-server/api/middleware/response.go

@@ -37,6 +37,14 @@ func ReturnError(c *gin.Context, statusCode, statusMessage string, data interfac
 	c.JSON(http.StatusOK, models.ResponseErrorJson{StatusCode: statusCode, StatusMessage: statusMessage, Data: data})
 }
 
+func BadRequestError(c *gin.Context, statusCode, statusMessage string, data interface{}) {
+	if data == nil {
+		data = []string{}
+	}
+	log.Logger.Error("Handle error", log.String("statusCode", statusCode), log.String("message", statusMessage))
+	c.JSON(http.StatusBadRequest, models.ResponseErrorJson{StatusCode: statusCode, StatusMessage: statusMessage, Data: data})
+}
+
 func ReturnBatchUpdateError(c *gin.Context, data []*models.ResponseErrorObj) {
 	ReturnError(c, "ERR_BATCH_CHANGE", "message", data)
 }
@@ -54,6 +62,11 @@ func ReturnServerHandleError(c *gin.Context, err error) {
 	ReturnError(c, "SERVER_HANDLE_ERROR", err.Error(), nil)
 }
 
+func ReturnBadRequestError(c *gin.Context, err error) {
+	log.Logger.Error("Bad Request error", log.Error(err))
+	BadRequestError(c, "BAD_REQUEST_ERROR", err.Error(), nil)
+}
+
 func ReturnTokenValidateError(c *gin.Context, err error) {
 	c.JSON(http.StatusUnauthorized, models.ResponseErrorJson{StatusCode: "TOKEN_VALIDATE_ERROR", StatusMessage: err.Error(), Data: nil})
 }

terraform-server/api/v1/provider/provider.go

@@ -129,3 +129,35 @@ func ProviderPluginImport(c *gin.Context) {
 		middleware.ReturnSuccess(c)
 	}
 }
+
+func ProviderDownload(c *gin.Context) {
+	id := c.Query("id")
+	if id == "" {
+		middleware.ReturnParamEmptyError(c, "id")
+		return
+	}
+	user := middleware.GetRequestUser(c)
+	err := db.ProviderDownload(id, user)
+	if err != nil {
+		middleware.ReturnBadRequestError(c, err)
+	} else {
+		middleware.ReturnSuccess(c)
+	}
+	return
+}
+
+func ProviderUpload(c *gin.Context) {
+	id := c.Query("id")
+	if id == "" {
+		middleware.ReturnParamEmptyError(c, "id")
+		return
+	}
+	user := middleware.GetRequestUser(c)
+	err := db.ProviderUpload(id, c.Request, user)
+	if err != nil {
+		middleware.ReturnBadRequestError(c, err)
+	} else {
+		middleware.ReturnSuccess(c)
+	}
+	return
+}

terraform-server/api/v1/resource_data/resource_data.go

@@ -12,7 +12,6 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
-	"sync"
 )
 
 func ResourceDataBatchCreate(c *gin.Context) {
@@ -31,6 +30,7 @@ func ResourceDataBatchCreate(c *gin.Context) {
 	}
 }
 
+/*
 func ResourceDataList(c *gin.Context) {
 	paramsMap := make(map[string]interface{})
 	rowData, err := db.ResourceDataList(paramsMap)
@@ -44,6 +44,24 @@ func ResourceDataList(c *gin.Context) {
 	}
 	return
 }
+*/
+
+func ResourceDataList (c *gin.Context) {
+	ids := c.Query("ids")
+	trimIds := strings.Trim(ids, ",")
+	queryIds := strings.Split(trimIds, ",")
+	queryIdsStr := strings.Join(queryIds, "','")
+	rowData, err := db.ResourceDataList(queryIdsStr)
+	if err != nil {
+		middleware.ReturnServerHandleError(c, err)
+	} else {
+		if len(rowData) == 0 {
+			rowData = []*models.ResourceDataQuery{}
+		}
+		middleware.ReturnData(c, rowData)
+	}
+	return
+}
 
 func ResourceDataBatchDelete(c *gin.Context) {
 	ids := c.Query("ids")
@@ -79,6 +97,54 @@ func ResourceDataBatchUpdate(c *gin.Context) {
 	return
 }
 
+func operationConsumer(ch chan int, done chan bool, request_param map[string]interface{}, params []map[string]interface{},
+	plugin string, action string, rowData *models.PluginInterfaceResultObj, resultChan chan []map[string]interface{}) {
+	for {
+		i, ok := <- ch
+		if ok {
+			if _, ok := request_param["operator"]; ok {
+				params[i]["operator_user"] = request_param["operator"]
+			} else {
+				params[i]["operator_user"] = "system"
+			}
+			params[i]["requestId"] = request_param["requestId"].(string) + "_" + strconv.Itoa(i + 1)
+			params[i]["requestSn"] = strconv.Itoa(i + 1)
+			debugFileContent := []map[string]interface{}{}
+			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent)
+			if _, ok := retData["errorCode"]; ok && retData["errorCode"] != "0" {
+				rowData.ResultCode = "1"
+				rowData.ResultMessage = "fail"
+			}
+			curResultOutputs := []map[string]interface{}{}
+			// handle one input, many output
+			if v, ok := retData[models.TerraformOutPutPrefix]; ok {
+				tmpData, _ := json.Marshal(v)
+				var resultList []map[string]interface{}
+				json.Unmarshal(tmpData, &resultList)
+				for i := range resultList {
+					tmpRetData := make(map[string]interface{})
+					tmpRetData["callbackParameter"] = retData["callbackParameter"]
+					tmpRetData["errorCode"] = retData["errorCode"]
+					tmpRetData["errorMessage"] = retData["errorMessage"]
+					for k, v := range resultList[i] {
+						tmpRetData[k] = v
+					}
+					// rowData.Results.Outputs = append(rowData.Results.Outputs, tmpRetData)
+					curResultOutputs = append(curResultOutputs, tmpRetData)
+				}
+			} else {
+				// rowData.Results.Outputs = append(rowData.Results.Outputs, retData)
+				curResultOutputs = append(curResultOutputs, retData)
+			}
+			resultChan<-curResultOutputs
+		} else {
+			break
+		}
+	}
+	done <- true
+	return
+}
+
 func TerraformOperation(c *gin.Context) {
 	rowData := models.PluginInterfaceResultObj{}
 	rowData.ResultCode = "0"
@@ -123,9 +189,32 @@ func TerraformOperation(c *gin.Context) {
 	// rowData := models.PluginInterfaceResultObj{}
 	// rowData.ResultCode = "0"
 	// rowData.ResultMessage = "success"
-	var curProviderData = models.ProviderTable{Name: ""}
 	count := len(params)
 	resultChan := make(chan []map[string]interface{}, count)
+	consumerCnt := models.ConsumerCount
+	if models.Config.RequestConsumerCount > 0 {
+		consumerCnt = models.Config.RequestConsumerCount
+	}
+	if count < consumerCnt {
+		consumerCnt = count
+	}
+	ch := make(chan int, consumerCnt)
+	doneChan := make(chan bool, consumerCnt)
+	for i := 0; i < consumerCnt; i++ {
+		go operationConsumer(ch, doneChan, request_param, params, plugin, action, &rowData, resultChan)
+	}
+
+	// producer
+	for i := 0; i < count; i++ {
+		ch <- i
+	}
+	close(ch)
+
+	for i := 0; i < consumerCnt; i++ {
+		<- doneChan
+	}
+
+	/*
 	var wg sync.WaitGroup
 	wg.Add(count)
 	for i := range params {
@@ -140,7 +229,7 @@ func TerraformOperation(c *gin.Context) {
 			params[i]["requestId"] = request_param["requestId"].(string) + "_" + strconv.Itoa(i + 1)
 			params[i]["requestSn"] = strconv.Itoa(i + 1)
 			debugFileContent := []map[string]interface{}{}
-			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent, &curProviderData)
+			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent)
 			if _, ok := retData["errorCode"]; ok && retData["errorCode"] != "0" {
 				rowData.ResultCode = "1"
 				rowData.ResultMessage = "fail"
@@ -170,19 +259,13 @@ func TerraformOperation(c *gin.Context) {
 		}(i)
 	}
 	wg.Wait()
+	 */
 	close(resultChan)
 	for i := range resultChan {
 		curRes := i
 		rowData.Results.Outputs = append(rowData.Results.Outputs, curRes...)
 	}
 
-	// clear the workpath
-	/*
-	if curProviderData.Name != "" {
-		db.DelDir(models.Config.TerraformFilePath + curProviderData.Name)
-	}
-	 */
-
 	c.JSON(http.StatusOK, rowData)
 	return
 }
@@ -251,9 +334,31 @@ func TerraformOperationDebug (c *gin.Context) {
 	// rowData.StatusCode = "OK"
 	// rowData.ResultCode = "0"
 	// rowData.ResultMessage = "success"
-	var curProviderData = models.ProviderTable{Name: ""}
 	count := len(params)
 	resultChan := make(chan map[string]interface{}, count)
+	consumerCnt := models.ConsumerCount
+	if models.Config.RequestConsumerCount > 0 {
+		consumerCnt = models.Config.RequestConsumerCount
+	}
+	if count < consumerCnt {
+		consumerCnt = count
+	}
+	ch := make(chan int, consumerCnt)
+	doneChan := make(chan bool, consumerCnt)
+	for i := 0; i < consumerCnt; i++ {
+		go operationDebugConsumer(ch, doneChan, request_param, params, plugin, action, &rowData, resultChan)
+	}
+
+	// producer
+	for i := 0; i < count; i++ {
+		ch <- i
+	}
+	close(ch)
+
+	for i := 0; i < consumerCnt; i++ {
+		<- doneChan
+	}
+	/*
 	var wg sync.WaitGroup
 	wg.Add(count)
 	for i := range params {
@@ -269,7 +374,7 @@ func TerraformOperationDebug (c *gin.Context) {
 			params[i]["requestSn"] = strconv.Itoa(i + 1)
 			params[i][models.ResourceDataDebug] = true
 			debugFileContent := []map[string]interface{}{}
-			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent, &curProviderData)
+			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent)
 			if _, ok := retData["errorCode"]; ok && retData["errorCode"] != "0" {
 				rowData.ResultCode = "1"
 				rowData.ResultMessage = "fail"
@@ -308,6 +413,7 @@ func TerraformOperationDebug (c *gin.Context) {
 		// rowData.Results.Outputs = append(rowData.Results.Outputs, curCombineResult)
 	}
 	wg.Wait()
+	*/
 	close(resultChan)
 	for i := range resultChan {
 		curRes := i
@@ -318,4 +424,60 @@ func TerraformOperationDebug (c *gin.Context) {
 	tmpRetVal, _ := json.Marshal(rowData)
 	c.Data(http.StatusOK, "application/json", tmpRetVal)
 	return
+}
+
+func operationDebugConsumer(ch chan int, done chan bool, request_param map[string]interface{}, params []map[string]interface{},
+	plugin string, action string, rowData *models.PluginInterfaceResultObjDebug, resultChan chan map[string]interface{}) {
+	for {
+		i, ok := <- ch
+		if ok {
+			if _, ok := request_param["operator"]; ok {
+				params[i]["operator_user"] = request_param["operator"]
+			} else {
+				params[i]["operator_user"] = "system"
+			}
+			params[i]["requestId"] = request_param["requestId"].(string) + "_" + strconv.Itoa(i+1)
+			params[i]["requestSn"] = strconv.Itoa(i + 1)
+			params[i][models.ResourceDataDebug] = true
+			debugFileContent := []map[string]interface{}{}
+			retData, _ := db.TerraformOperation(plugin, action, params[i], &debugFileContent)
+			if _, ok := retData["errorCode"]; ok && retData["errorCode"] != "0" {
+				rowData.ResultCode = "1"
+				rowData.ResultMessage = "fail"
+			}
+
+			// handle one input, many output
+			curResultOutputs := []map[string]interface{}{}
+			if v, ok := retData[models.TerraformOutPutPrefix]; ok {
+				tmpData, _ := json.Marshal(v)
+				var resultList []map[string]interface{}
+				json.Unmarshal(tmpData, &resultList)
+				for i := range resultList {
+					tmpRetData := make(map[string]interface{})
+					tmpRetData["callbackParameter"] = retData["callbackParameter"]
+					tmpRetData["errorCode"] = retData["errorCode"]
+					tmpRetData["errorMessage"] = retData["errorMessage"]
+					for k, v := range resultList[i] {
+						tmpRetData[k] = v
+					}
+					curResultOutputs = append(curResultOutputs, tmpRetData)
+				}
+				if len(resultList) == 0 {
+					delete(retData, models.TerraformOutPutPrefix)
+					curResultOutputs = append(curResultOutputs, retData)
+				}
+			} else {
+				curResultOutputs = append(curResultOutputs, retData)
+			}
+			curCombineResult := make(map[string]interface{})
+			curCombineResult["result_data"] = curResultOutputs
+
+			curCombineResult["resource_results"] = debugFileContent
+			resultChan<-curCombineResult
+		} else {
+			break
+		}
+	}
+	done <- true
+	return
 }

terraform-server/conf/default.json

@@ -37,5 +37,6 @@
   "terraform_file_path": "/data/terraform/",
   "terraform_cmd_path": "/usr/local/bin/terraform",
   "terraform_provider_os_arch": "linux_amd64",
-  "version": ""
+  "version": "",
+  "request_consumer_count": 6
 }

terraform-server/models/config.go

@@ -71,6 +71,7 @@ type GlobalConfig struct {
 	TerraformCmdPath        string                        `json:"terraform_cmd_path"`
 	TerraformProviderOsArch string                        `json:"terraform_provider_os_arch"`
 	Version                 string                        `json:"version"`
+	RequestConsumerCount    int                           `json:"request_consumer_count"`
 	// default json
 }
 

terraform-server/models/const.go

@@ -20,6 +20,8 @@ const (
 	ParameterSourceDefault = "custom"
 	TfArgumentKeyArgumentDefault = "N"
 	CommandTimeOut      = 300
+	DownloadProviderTimeOut = 1800
+	ConsumerCount       = 5
 	ResourceDataDebug   = "$_resource_data_debug$"
 	ResourceIdDataConvert  = "#resourceId#"
 	PGuid               = "6101d5ff9c058ecd8d2dddd974d38f98"
@@ -32,6 +34,7 @@ const (
 
 var (
 	SEPERATOR  = string([]byte{0x01})
+	ProviderName = map[string]string{"Aws":"aws", "Alicloud":"alicloud", "Tencentcloud":"tencentcloud"}
 	ConvertWay = map[string]string{"Data": "data", "Template": "template", "ContextData": "context_data", "Attr": "attribute", "Direct": "direct", "Function": "function", "ContextDirect": "context_direct", "ContextAttr": "context_attribute", "ContextTemplate": "context_template"}
 	// TerraformProviderPathDiffMap = map[string]string{"tencentcloud": ".terraform/providers/registry.terraform.io/tencentcloudstack/tencentcloud/",
 	// 												 "alicloud": ".terraform/providers/registry.terraform.io/hashicorp/alicloud/"}