Skip to content

Commit

Permalink
feat: add support to delete cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
@rafatio
rafatio committed Nov 21, 2023
1 parent 41f7da8 commit 3eaa89f
Show file tree
Hide file tree
Showing 13 changed files with 458 additions and 78 deletions.
3 changes: 3 additions & 0 deletions apis/controlplane/v1alpha1/kopscontrolplane_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ const (

// KopsControlPlaneFinalizer allows the controller to clean up resources on delete.
KopsControlPlaneFinalizer = "kopscontrolplane.controlplane.cluster.x-k8s.io"

// ClusterNameAnnotation is the annotation set on KopsControlPlane to signalize that it shouldn't be deleted even with deletionTimestamp.
ClusterDeleteProtectionAnnotation = "cluster.x-k8s.io/delete-protection"
)

const (
Expand Down
166 changes: 121 additions & 45 deletions controllers/controlplane/kopscontrolplane_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/external"
capiutil "sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/kubeconfig"
Expand Down Expand Up @@ -98,6 +97,8 @@ type KopsControlPlaneReconciler struct {
PopulateClusterSpecFactory func(ctx context.Context, kopsCluster *kopsapi.Cluster, kopsClientset simple.Clientset, cloud fi.Cloud) (*kopsapi.Cluster, error)
PrepareKopsCloudResourcesFactory func(ctx context.Context, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster, terraformOutputDir string, cloud fi.Cloud) error
ApplyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
DestroyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
KopsDeleteResourcesFactory func(ctx context.Context, cloud fi.Cloud, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster) error
ValidateKopsClusterFactory func(kubeConfig *rest.Config, kopsCluster *kopsapi.Cluster, cloud fi.Cloud, igs *kopsapi.InstanceGroupList) (*validation.ValidationCluster, error)
GetClusterStatusFactory func(kopsCluster *kopsapi.Cluster, cloud fi.Cloud) (*kopsapi.ClusterStatus, error)
GetASGByNameFactory func(kopsMachinePool *infrastructurev1alpha1.KopsMachinePool, kopsControlPlane *controlplanev1alpha1.KopsControlPlane, credentials *aws.Credentials) (*asgTypes.AutoScalingGroup, error)
Expand All @@ -124,14 +125,6 @@ func init() {
klog.SetLogger(log)
}

func ApplyTerraform(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error {
err := utils.ApplyTerraform(ctx, terraformDir, tfExecPath, credentials)
if err != nil {
return err
}
return nil
}

func GetClusterStatus(kopsCluster *kopsapi.Cluster, cloud fi.Cloud) (*kopsapi.ClusterStatus, error) {
status, err := cloud.FindClusterStatus(kopsCluster)
if err != nil {
Expand All @@ -140,6 +133,17 @@ func GetClusterStatus(kopsCluster *kopsapi.Cluster, cloud fi.Cloud) (*kopsapi.Cl
return status, nil
}

func (r *KopsControlPlaneReconciler) shouldDeleteCluster(kcp *controlplanev1alpha1.KopsControlPlane) bool {
if !kcp.ObjectMeta.DeletionTimestamp.IsZero() {
if kcp.Annotations[controlplanev1alpha1.ClusterDeleteProtectionAnnotation] == "true" {
r.Recorder.Eventf(kcp, corev1.EventTypeWarning, "ClusterDeleteProtectionEnabled", "cluster delete protection is enabled, skipping deletion")
return false
}
return true
}
return false
}

func (r *KopsControlPlaneReconciler) PrepareCustomCloudResources(ctx context.Context, kopsCluster *kopsapi.Cluster, kopsControlPlane *controlplanev1alpha1.KopsControlPlane, kmps []infrastructurev1alpha1.KopsMachinePool, shouldEnableKarpenter bool, configBase, terraformOutputDir string, shouldIgnoreSG bool) error {
s3Bucket, err := utils.GetBucketName(configBase)
if err != nil {
Expand Down Expand Up @@ -549,29 +553,42 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req

var kmps []infrastructurev1alpha1.KopsMachinePool

terraformOutputDir := fmt.Sprintf("/tmp/%s", owner.GetName())

// Attempt to Update the KopsControlPlane and KopsMachinePool object and status after each reconciliation if no error occurs.
defer func() {
if shouldUnlock {
reconciler.Mux.Unlock()
reconciler.log.Info(fmt.Sprintf("unexpected Unlock step for %s, took %s", kopsControlPlane.Name, time.Since(lockInitTime)))
}
if err := reconciler.Status().Update(ctx, kopsControlPlane); err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, controlplanev1alpha1.FailedToUpdateKopsControlPlane, "failed to update kopsControlPlane: %s", err)
}

for _, kopsMachinePool := range kmps {
kopsMachinePoolHelper := kopsMachinePool.DeepCopy()
if err := reconciler.Update(ctx, &kopsMachinePool); err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, controlplanev1alpha1.FailedToUpdateKopsControlPlane, "failed to update kopsControlPlane: %s", err)
r.Recorder.Eventf(&kopsMachinePool, corev1.EventTypeWarning, infrastructurev1alpha1.FailedToUpdateKopsMachinePool, "failed to update kopsMachinePool: %s", err)
}

if kopsMachinePool.ObjectMeta.DeletionTimestamp.IsZero() {
if kopsMachinePoolHelper.ObjectMeta.DeletionTimestamp.IsZero() && kopsControlPlane.ObjectMeta.DeletionTimestamp.IsZero() {
kopsMachinePool.Status = kopsMachinePoolHelper.Status
if err := reconciler.Status().Update(ctx, &kopsMachinePool); err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, infrastructurev1alpha1.FailedToUpdateKopsMachinePool, "failed to update kopsMachinePool: %s", err)
r.Recorder.Eventf(&kopsMachinePool, corev1.EventTypeWarning, infrastructurev1alpha1.FailedToUpdateKopsMachinePool, "failed to update kopsMachinePool: %s", err)
}
}
}

kopsControlPlaneHelper := kopsControlPlane.DeepCopy()
if err := reconciler.Update(ctx, kopsControlPlane); err != nil {
reconciler.log.Info(fmt.Sprintf("%+v", kopsControlPlane))
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, controlplanev1alpha1.FailedToUpdateKopsControlPlane, "failed to update kopsControlPlane: %s", err)
}

if kopsControlPlaneHelper.ObjectMeta.DeletionTimestamp.IsZero() {
kopsControlPlane.Status = kopsControlPlaneHelper.Status
if err := reconciler.Status().Update(ctx, kopsControlPlane); err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, controlplanev1alpha1.FailedToUpdateKopsControlPlane, "failed to update kopsControlPlane: %s", err)
}
}

err := utils.CleanupTerraformDirectory(terraformOutputDir)
if err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedCleanupTerraformDirectory", "failed to cleanup terraform directory from cluster: %s", err)
Expand All @@ -594,11 +611,96 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
reconciler.awsCredentials = *awsCredentials

kopsClientset, err := reconciler.GetKopsClientSetFactory(kopsControlPlane.Spec.KopsClusterSpec.ConfigBase)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedInstantiateKopsClient", "failed to instantiate Kops client: %s", err)
return resultError, err
}

kopsCluster := &kopsapi.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: owner.GetName(),
},
Spec: kopsControlPlane.Spec.KopsClusterSpec,
}

kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, reconciler.Client, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
if err != nil {
return resultError, err
}

if r.shouldDeleteCluster(kopsControlPlane) {
log.Info(fmt.Sprintf("deleting cluster %s", owner.GetName()))

kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, reconciler.Client, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
if err != nil {
return resultError, err
}

err = reconciler.PrepareCustomCloudResources(ctx, kopsCluster, kopsControlPlane, nil, false, kopsControlPlane.Spec.KopsClusterSpec.ConfigBase, terraformOutputDir, false)
if err != nil {
return resultError, err
}

err = utils.CreateTerraformFilesFromTemplate("templates/provider.tf.tpl", "provider.tf", terraformOutputDir, nil)
if err != nil {
return resultError, err
}

err = reconciler.DestroyTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToDestroyTerraform", "failed to destroy terraform: %s", err)
return resultError, err
}
reconciler.Mux.Lock()
err = util.SetEnvVarsFromAWSCredentials(reconciler.awsCredentials)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToSetAWSEnvVars", "failed to set AWS environment variables: %s", err)
return resultError, err
}

cloud, err := reconciler.BuildCloudFactory(kopsCluster)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToBuildCloudConfig", "failed to build Cloud Config: %s", err)
return resultError, err
}

reconciler.Mux.Unlock()

err = r.KopsDeleteResourcesFactory(ctx, cloud, kopsClientset, kopsCluster)
if err != nil {
return resultError, err
}

for i := range kmps {

err = util.DeleteOwnerResources(ctx, r.Client, &kmps[i])
if err != nil {
return resultError, err
}

// This is needed because the owner deletion triggers an update in the resource
err = reconciler.Get(ctx, types.NamespacedName{Name: kmps[i].Name, Namespace: kmps[i].Namespace}, &kmps[i])
if err != nil {
return resultError, err
}

controllerutil.RemoveFinalizer(&kmps[i], infrastructurev1alpha1.KopsMachinePoolFinalizer)
}

err = util.DeleteOwnerResources(ctx, r.Client, kopsControlPlane)
if err != nil {
return resultError, err
}

controllerutil.RemoveFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer)
return resultDefault, nil
}

if !controllerutil.ContainsFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer) {
controllerutil.AddFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer)
}

reconciler.Mux.Lock()
shouldUnlock = true

Expand All @@ -611,12 +713,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, err
}

kopsClientset, err := reconciler.GetKopsClientSetFactory(kopsControlPlane.Spec.KopsClusterSpec.ConfigBase)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedInstantiateKopsClient", "failed to instantiate Kops client: %s", err)
return resultError, err
}

shouldEnableKarpenter := false
existingKopsMachinePool := []infrastructurev1alpha1.KopsMachinePool{}
for i, kopsMachinePool := range kmps {
Expand All @@ -642,13 +738,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
}

kopsCluster := &kopsapi.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: owner.GetName(),
},
Spec: kopsControlPlane.Spec.KopsClusterSpec,
}

err = reconciler.reconcileClusterAddons(ctx, kopsClientset, kopsCluster)
if err != nil {
return resultError, err
Expand Down Expand Up @@ -693,8 +782,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
conditions.MarkTrue(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition)

terraformOutputDir := fmt.Sprintf("/tmp/%s", kopsCluster.Name)

var shouldIgnoreSG bool
if _, ok := owner.GetAnnotations()["kopscontrolplane.controlplane.wildlife.io/external-security-groups"]; ok {
shouldIgnoreSG = true
Expand Down Expand Up @@ -823,25 +910,14 @@ func (r *KopsControlPlaneReconciliation) reconcileKopsMachinePool(ctx context.Co

if !kopsMachinePool.ObjectMeta.DeletionTimestamp.IsZero() {
r.log.Info(fmt.Sprintf("deleting instancegroup/%s", kopsInstanceGroup.Name))

for _, ownerReference := range kopsMachinePool.GetOwnerReferences() {
err := r.Client.Delete(ctx, capiutil.ObjectReferenceToUnstructured(
corev1.ObjectReference{
Kind: ownerReference.Kind,
Namespace: kopsMachinePool.Namespace,
Name: ownerReference.Name,
UID: ownerReference.UID,
APIVersion: ownerReference.APIVersion,
},
))
if client.IgnoreNotFound(err) != nil {
return err
}
}
err := kopsClientset.InstanceGroupsFor(kopsCluster).Delete(ctx, kopsInstanceGroup.Name, metav1.DeleteOptions{})
if err != nil {
return err
}
err = util.DeleteOwnerResources(ctx, r.Client, kopsMachinePool)
if err != nil {
return err
}
r.Recorder.Eventf(kopsMachinePool, corev1.EventTypeNormal, "KopsMachinePoolDeleted", kopsMachinePool.Name)
controllerutil.RemoveFinalizer(kopsMachinePool, infrastructurev1alpha1.KopsMachinePoolFinalizer)
return nil
Expand Down
Loading

0 comments on commit 3eaa89f

Please sign in to comment.