Retry cloud connection create/update when vpc is unavailable (#4766)

* Retry cloud connection create/update when vpc is unavailable

* review comment code changes

* review comment code changes

* review comment code changes

* review comment code changes

go.mod

@@ -5,7 +5,7 @@ go 1.18
 require (
 	github.com/IBM-Cloud/bluemix-go v0.0.0-20230601050310-eecebfbff63e
 	github.com/IBM-Cloud/container-services-go-sdk v0.0.0-20230118060037-101bda076037
-	github.com/IBM-Cloud/power-go-client v1.2.2
+	github.com/IBM-Cloud/power-go-client v1.2.4
 	github.com/IBM/apigateway-go-sdk v0.0.0-20210714141226-a5d5d49caaca
 	github.com/IBM/appconfiguration-go-admin-sdk v0.3.0
 	github.com/IBM/appid-management-go-sdk v0.0.0-20210908164609-dd0e0eaf732f

go.sum

@@ -105,6 +105,8 @@ github.com/IBM-Cloud/container-services-go-sdk v0.0.0-20230118060037-101bda07603
 github.com/IBM-Cloud/ibm-cloud-cli-sdk v0.5.3/go.mod h1:RiUvKuHKTBmBApDMUQzBL14pQUGKcx/IioKQPIcRQjs=
 github.com/IBM-Cloud/power-go-client v1.2.2 h1:VNlzizoG2x06c3nL1ZBILF701QcvXcu6nEH3hmEKCkw=
 github.com/IBM-Cloud/power-go-client v1.2.2/go.mod h1:Qfx0fNi+9hms+xu9Z6Euhu9088ByW6C/TCMLECTRWNE=
+github.com/IBM-Cloud/power-go-client v1.2.4 h1:4y/ubiOXpMg3xyBryfgfsa8hae/9Dn5WLdvphoxvgsQ=
+github.com/IBM-Cloud/power-go-client v1.2.4/go.mod h1:0YVWoIQN5I5IvyhO/m4yxgPJqCh9QjceN2FNlVpYlOQ=
 github.com/IBM-Cloud/softlayer-go v1.0.5-tf h1:koUAyF9b6X78lLLruGYPSOmrfY2YcGYKOj/Ug9nbKNw=
 github.com/IBM-Cloud/softlayer-go v1.0.5-tf/go.mod h1:6HepcfAXROz0Rf63krk5hPZyHT6qyx2MNvYyHof7ik4=
 github.com/IBM/apigateway-go-sdk v0.0.0-20210714141226-a5d5d49caaca h1:crniVcf+YcmgF03NmmfonXwSQ73oJF+IohFYBwknMxs=

ibm/service/power/resource_ibm_pi_cloud_connection.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"regexp"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -22,6 +23,15 @@ import (
 	"github.com/IBM-Cloud/terraform-provider-ibm/ibm/validate"
 )
 
+var (
+	vpcUnavailable = regexp.MustCompile("pcloudCloudconnectionsPostServiceUnavailable|pcloudCloudconnectionsPutServiceUnavailable")
+)
+
+const (
+	vpcRetryCount    = 2
+	vpcRetryDuration = time.Minute
+)
+
 func ResourceIBMPICloudConnection() *schema.Resource {
 	return &schema.Resource{
 		CreateContext: resourceIBMPICloudConnectionCreate,
@@ -226,8 +236,16 @@ func resourceIBMPICloudConnectionCreate(ctx context.Context, d *schema.ResourceD
 	client := st.NewIBMPICloudConnectionClient(ctx, sess, cloudInstanceID)
 	cloudConnection, cloudConnectionJob, err := client.Create(body)
 	if err != nil {
-		log.Printf("[DEBUG] create cloud connection failed %v", err)
-		return diag.FromErr(err)
+		if vpcUnavailable.Match([]byte(err.Error())) {
+			err = retryCloudConnectionsVPC(func() (err error) {
+				cloudConnection, cloudConnectionJob, err = client.Create(body)
+				return
+			}, "create", err)
+		}
+		if err != nil {
+			log.Printf("[DEBUG] create cloud connection failed %v", err)
+			return diag.FromErr(err)
+		}
 	}
 
 	if cloudConnection != nil {
@@ -333,7 +351,16 @@ func resourceIBMPICloudConnectionUpdate(ctx context.Context, d *schema.ResourceD
 
 		_, cloudConnectionJob, err := client.Update(cloudConnectionID, body)
 		if err != nil {
-			return diag.FromErr(err)
+			if vpcUnavailable.Match([]byte(err.Error())) {
+				err = retryCloudConnectionsVPC(func() (err error) {
+					_, cloudConnectionJob, err = client.Update(cloudConnectionID, body)
+					return
+				}, "update", err)
+			}
+			if err != nil {
+				log.Printf("[DEBUG] update cloud connection failed %v", err)
+				return diag.FromErr(err)
+			}
 		}
 		if cloudConnectionJob != nil {
 			_, err = waitForIBMPIJobCompleted(ctx, jobClient, *cloudConnectionJob.ID, d.Timeout(schema.TimeoutCreate))
@@ -497,3 +524,13 @@ func resourceIBMPICloudConnectionDelete(ctx context.Context, d *schema.ResourceD
 	d.SetId("")
 	return nil
 }
+
+func retryCloudConnectionsVPC(ccVPCRetry func() error, operation string, errMsg error) error {
+	for count := 0; count < vpcRetryCount && errMsg != nil; count++ {
+		log.Printf("[DEBUG] unable to get vpc details for cloud connection: %v", errMsg)
+		time.Sleep(vpcRetryDuration)
+		log.Printf("[DEBUG] retrying cloud connection %s, retry #%v", operation, count+1)
+		errMsg = ccVPCRetry()
+	}
+	return errMsg
+}