[integ-tests-framework] Retry deletion of VPC stacks

VPC deletion failure is sporadic caused by resources in the VPC not fully deleted. Retrying deletion give more time for the resources to be deleted and VPC deleted successfully.

Other resources deletions are not retried,because we want to investigate those failures.

Signed-off-by: Hanwen <[email protected]>

tests/integration-tests/cfn_stacks_factory.py

@@ -304,7 +304,7 @@ def update_stack(
                     "Couldn't find stack with name {0} in region {1}. Skipping update.".format(name, region)
                 )
 
-    def delete_all_stacks(self, excluded_stacks=None):
+    def delete_all_stacks(self, excluded_stacks=None):  # noqa: C901
         """Destroy all created stacks except for those in excluded_stacks."""
         logging.debug("Destroying all cfn stacks")
         for value in reversed(OrderedDict(self.__created_stacks).values()):
@@ -319,11 +319,27 @@ def delete_all_stacks(self, excluded_stacks=None):
             try:
                 self.delete_stack(value.name, value.region)
             except Exception as e:
-                logging.error(
-                    "Failed when destroying stack {0} in region {1} with exception {2}.".format(
-                        value.name, value.region, e
+                if "-vpc-" in value.name:
+                    # Retry deletion only if it is a VPC stack.
+                    # Because VPC stack is not part of the released product, we can ignore deletion failures.
+                    logging.warning(
+                        "Failed when destroying stack {0} in region {1} with exception {2}. "
+                        "Trying delete again.".format(value.name, value.region, e)
+                    )
+                    try:
+                        self.delete_stack(value.name, value.region)
+                    except Exception as e:
+                        logging.error(
+                            "Failed when destroying stack {0} in region {1} with exception {2}.".format(
+                                value.name, value.region, e
+                            )
+                        )
+                else:
+                    logging.error(
+                        "Failed when destroying stack {0} in region {1} with exception {2}.".format(
+                            value.name, value.region, e
+                        )
                     )
-                )
 
     @retry(
         retry_on_result=lambda result: result == "CREATE_IN_PROGRESS",