Skip to content

Commit

Permalink
fix: add --force-remove-finalizers to platform destroy
Browse files Browse the repository at this point in the history 
Signed-off-by: Rohan CJ <[email protected]>
(cherry picked from commit df5b81e)
  • Loading branch information
@rohantmp @cbron
rohantmp authored and cbron committed Dec 18, 2024
1 parent 9472510 commit 31e94e2
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 37 deletions.
20 changes: 17 additions & 3 deletions cmd/vclusterctl/cmd/platform/destroy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ before running this command:
2. Helm v3 must be installed
VirtualClusterInstances managed with driver helm will be deleted, but the underlying virtual cluster will not be uninstalled
VirtualClusterInstances managed with driver helm will be deleted, but the underlying virtual cluster will not be uninstalled.
########################################################
`,
Expand All @@ -69,7 +69,8 @@ VirtualClusterInstances managed with driver helm will be deleted, but the underl
destroyCmd.Flags().BoolVar(&cmd.IgnoreNotFound, "ignore-not-found", false, "Exit successfully if platform installation is not found")
destroyCmd.Flags().BoolVar(&cmd.Force, "force", false, "Try uninstalling even if the platform is not installed. '--namespace' is required if true")
destroyCmd.Flags().BoolVar(&cmd.NonInteractive, "non-interactive", false, "Will not prompt for confirmation")
destroyCmd.Flags().IntVar(&cmd.TimeoutMinutes, "timeout-minutes", 5, "How long to try deleting the platform before giving up")
destroyCmd.Flags().IntVar(&cmd.TimeoutMinutes, "timeout-minutes", 5, "How long to try deleting the platform before giving up. May increase when removing finalizers if --remove-finalizers is used")
destroyCmd.Flags().BoolVar(&cmd.ForceRemoveFinalizers, "force-remove-finalizers", false, "IMPORTANT! Removing finalizers may cause unintended behaviours like leaving resources behind, but will ensure the platform is uninstalled.")

return destroyCmd
}
Expand Down Expand Up @@ -110,7 +111,7 @@ func (cmd *DestroyCmd) Run(ctx context.Context) error {
if terminal.IsTerminalIn {
deleteOpt := "delete"
out, err := cmd.Log.Question(&survey.QuestionOptions{
Question: fmt.Sprintf("IMPORTANT! You are destroy the vCluster Platform in the namespace %q.\nThis may result in data loss. Please ensure your kube-context is pointed at the right cluster.\n Please type %q to continue:", cmd.Namespace, deleteOpt),
Question: fmt.Sprintf("IMPORTANT! You are destroying the vCluster Platform installation in the namespace %q.\nThis may result in data loss. Please ensure your kube-context is pointed at the right cluster.\n Please type %q to continue:", cmd.Namespace, deleteOpt),
})
if err != nil {
return fmt.Errorf("failed to prompt for confirmation: %w", err)
Expand All @@ -119,6 +120,19 @@ func (cmd *DestroyCmd) Run(ctx context.Context) error {
cmd.Log.Info("destroy cancelled")
return nil
}
if cmd.ForceRemoveFinalizers {
forceRemoveOpt := "force-remove"
out, err := cmd.Log.Question(&survey.QuestionOptions{
Question: fmt.Sprintf("IMPORTANT! You have selected the --force-remove-finalizers option. Please ensure you understand the consequences. Removing finalizers may cause unintended behaviours like leaving resources behind, but will ensure the platform is uninstalled. To confirm, please type %q", forceRemoveOpt),
})
if err != nil {
return fmt.Errorf("failed to prompt for confirmation: %w", err)
}
if out != forceRemoveOpt {
cmd.Log.Info("destroy cancelled")
return nil
}
}
}

err = destroy.Destroy(ctx, cmd.DeleteOptions)
Expand Down
147 changes: 113 additions & 34 deletions pkg/cli/destroy/destroy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,23 @@ package destroy

import (
"context"
"errors"
"fmt"
"math"
"strings"
"time"

storagev1 "github.com/loft-sh/api/v4/pkg/apis/storage/v1"
"github.com/loft-sh/log"
"github.com/loft-sh/vcluster/pkg/cli/start"
"github.com/loft-sh/vcluster/pkg/platform/clihelper"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/discovery"
Expand All @@ -32,6 +36,7 @@ var resourceOrder = []string{
// templates
"virtualclustertemplates",
"devpodenvironmenttemplates",
"devpodworkspacepresets",
"devpodworkspacetemplates",
"clusterroletemplates",
"spacetemplates",
Expand Down Expand Up @@ -70,17 +75,26 @@ var legacyResources = []string{
type DeleteOptions struct {
start.Options
// cli options
DeleteNamespace bool
IgnoreNotFound bool
Force bool
NonInteractive bool
TimeoutMinutes int
DeleteNamespace bool
IgnoreNotFound bool
Force bool
ForceRemoveFinalizers bool
NonInteractive bool
TimeoutMinutes int
}

var backoffFactor = 1.2

func Destroy(ctx context.Context, opts DeleteOptions) error {
ctx, cancel := context.WithTimeout(ctx, time.Duration(opts.TimeoutMinutes)*time.Minute)
func Destroy(ctxWithoutTimeout context.Context, opts DeleteOptions) error {
err := destroy(ctxWithoutTimeout, opts)
if errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("timed out: %w", err)
}
return err
}

func destroy(ctxWithoutTimeout context.Context, opts DeleteOptions) error {
ctx, cancel := context.WithTimeout(ctxWithoutTimeout, time.Duration(opts.TimeoutMinutes)*time.Minute)
defer cancel()

// create time.Duration(opts.TimeoutMinutes) * time.Minutea dynamic client
Expand Down Expand Up @@ -140,11 +154,13 @@ func Destroy(ctx context.Context, opts DeleteOptions) error {
}
continue
}
// list and delete all resources
err = deleteAllResourcesAndWait(ctx, dynamicClient, opts.Log, opts.TimeoutMinutes, "storage.loft.sh", "v1", resourceName)
// list and delete all resources. If times out because of resources, the timeout will be repeated and new context will be created
ctx, cancel, err = deleteAllResourcesAndWait(ctxWithoutTimeout, ctx, dynamicClient, opts.Log, opts.ForceRemoveFinalizers, opts.TimeoutMinutes, "storage.loft.sh", "v1", resourceName)
defer cancel()
if err != nil {
return fmt.Errorf("failed to delete resource %q: %w", resourceName, err)
}
defer cancel()
}

// helm uninstall and others
Expand Down Expand Up @@ -240,38 +256,101 @@ func Destroy(ctx context.Context, opts DeleteOptions) error {
return nil
}

func deleteAllResourcesAndWait(ctx context.Context, dynamicClient dynamic.Interface, log log.Logger, timeoutMinutes int, group, version, resource string) error {
func deleteAllResourcesAndWait(ctxWithoutDeadline, ctxWithDeadLine context.Context, dynamicClient dynamic.Interface, log log.Logger, deleteFinalizers bool, timeoutMinutes int, group, version, resource string) (context.Context, context.CancelFunc, error) {
gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{Duration: time.Second, Factor: backoffFactor, Cap: time.Duration(timeoutMinutes) * time.Minute, Steps: math.MaxInt32}, func(ctx context.Context) (bool, error) {
log.Debugf("checking all %q", resource)

resourceClient := dynamicClient.Resource(gvr)
list, err := resourceClient.List(ctx, metav1.ListOptions{})
if err != nil {
return false, err
}
if len(list.Items) == 0 {
return true, nil
// function to poll with wait.ExponentialBackoffWithContext
deleteAndWait := func(deleteFinalizers bool) func(ctx context.Context) (bool, error) {
// log each key as waiting only once on the info levell, and continue logging on the debug level
loggedDeletion := sets.New[string]()
infofOnceThenDebugf := func(str string, args ...interface{}) {
logLine := fmt.Sprintf(str, args...)
if loggedDeletion.Has(logLine) {
log.Debug(logLine)
return
}
log.Info(logLine)
loggedDeletion.Insert(logLine)
}
for _, object := range list.Items {
if !object.GetDeletionTimestamp().IsZero() {
return false, nil

return func(ctx context.Context) (bool, error) {
infofOnceThenDebugf("checking all %q", resource)

// fetch all
resourceClient := dynamicClient.Resource(gvr)
list, err := resourceClient.List(ctx, metav1.ListOptions{})
if err != nil {
return false, err
}
if object.GetNamespace() == "" {
log.Infof("deleting %v: %v", resource, object.GetName())
} else {
log.Infof("deleting %v: %v/%v", resource, object.GetNamespace(), object.GetName())
// succeed when all resources are deleted
if len(list.Items) == 0 {
return true, nil
}
err := resourceClient.Namespace(object.GetNamespace()).Delete(ctx, object.GetName(), metav1.DeleteOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return false, err

isVCluster := resource == "virtualclusterinstances"

// delete all resources and log deleting resources
for _, object := range list.Items {
// get namespaced name
namespacedName := object.GetName()
namespace := object.GetNamespace()
if namespace != "" {
namespacedName += "/" + namespace
}
isExternalVCluster := false
if isVCluster {
//convert unstructured to VirtualClusterInstance
virtualClusterInstance := &storagev1.VirtualClusterInstance{}
err = runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &virtualClusterInstance)
if err != nil {
log.Warnf("couldn't cast %q object %q to VirtualClusterInstance: %v", resource, namespacedName, err)
}
isExternalVCluster = virtualClusterInstance.Spec.External
}
// delete object if not already deleted
if object.GetDeletionTimestamp().IsZero() {
if !isExternalVCluster {
log.Infof("deleting %v: %q", resource, namespacedName)
} else {
log.Infof("deleting externally deployed %v, the virtual cluster itself will remain: %q", resource, namespacedName)
}
err := resourceClient.Namespace(object.GetNamespace()).Delete(ctx, object.GetName(), metav1.DeleteOptions{})
if kerrors.IsNotFound(err) {
continue
} else if err != nil {
return false, err
}
} else {
infofOnceThenDebugf("deleted resource found, waiting for cleanup: %v", object.GetName())
}
// object exists and delete command succeeded
if deleteFinalizers {
log.Infof("removing finalizers from %v: %q", resource, namespacedName)
_, err = resourceClient.Namespace(object.GetNamespace()).Patch(ctx, object.GetName(), types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return false, err
}
}
}
return false, nil
}
return false, nil
})
if err != nil {
return err
}
err := wait.ExponentialBackoffWithContext(ctxWithDeadLine, wait.Backoff{Duration: time.Second, Factor: backoffFactor, Cap: time.Duration(timeoutMinutes) * time.Minute, Steps: math.MaxInt32}, deleteAndWait(false))
if !errors.Is(err, context.DeadlineExceeded) {
// return the err unless timed out. If timed out, remove finalizers and retry
return ctxWithDeadLine, func() {}, err
}

return nil
// the timeout is hit, begin removing finalizers and rety
if !deleteFinalizers {
return ctxWithDeadLine, func() {}, fmt.Errorf("timed out waiting for %q to be deleted", resource)
}
// new context now that the old deadline is exceeded
ctx, cancel := context.WithTimeout(ctxWithoutDeadline, time.Duration(timeoutMinutes)*time.Minute)
log.Warn("operation timed out. Removing finalizers from stuck resources, resetting timeout")
err = wait.ExponentialBackoffWithContext(ctxWithoutDeadline, wait.Backoff{Duration: time.Second, Factor: backoffFactor, Cap: time.Duration(timeoutMinutes) * time.Minute, Steps: math.MaxInt32}, deleteAndWait(true))
if errors.Is(err, context.DeadlineExceeded) {
return ctx, cancel, fmt.Errorf("timed out waiting for %q to be deleted", resource)
}
return ctx, cancel, err
}

0 comments on commit 31e94e2

Please sign in to comment.