L4LBMeta CR's finalizer (#75)

Bug Fixes:
- Added a finalizer to L4LBMeta CRs to address a bug where L4LBs were not properly deleted when a namespace was deleted due to issues arising from the L4LBMetas already being removed

.gitignore

@@ -26,3 +26,4 @@ k8s
 .netrc
 testbin
 config/samples
+.vscode

controllers/l4lb_controller.go

@@ -29,7 +29,6 @@ import (
 
 	k8sv1alpha1 "github.com/netrisai/netris-operator/api/v1alpha1"
 	"github.com/netrisai/netris-operator/netrisstorage"
-	"github.com/netrisai/netriswebapi/http"
 	api "github.com/netrisai/netriswebapi/v2"
 )
 
@@ -94,13 +93,15 @@ func (r *L4LBReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 
 	if l4lb.DeletionTimestamp != nil {
 		logger.Info("Go to delete")
-		_, err := r.deleteL4LB(l4lb, l4lbMeta)
+		result, err := r.deleteL4LB(l4lb, l4lbMeta)
 		if err != nil {
 			logger.Error(fmt.Errorf("{deleteL4LB} %s", err), "")
 			return u.patchL4LBStatus(l4lb, "Failure", err.Error())
 		}
-		logger.Info("L4LB deleted")
-		return ctrl.Result{}, nil
+		if result.IsZero() {
+			logger.Info("L4LB deleted")
+		}
+		return result, nil
 	}
 
 	if l4lbMustUpdateAnnotations(l4lb) {
@@ -159,6 +160,7 @@ func (r *L4LBReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		}
 
 		l4lbMeta.Spec.L4LBCRGeneration = l4lb.GetGeneration()
+		l4lbMeta.SetFinalizers([]string{"resource.k8s.netris.ai/delete"})
 
 		l4lbCreateCtx, l4lbCreateCancel := context.WithTimeout(cntxt, contextTimeout)
 		defer l4lbCreateCancel()
@@ -172,19 +174,6 @@ func (r *L4LBReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 }
 
 func (r *L4LBReconciler) deleteL4LB(l4lb *k8sv1alpha1.L4LB, l4lbMeta *k8sv1alpha1.L4LBMeta) (ctrl.Result, error) {
-	if l4lbMeta != nil && l4lbMeta.Spec.ID > 0 && !l4lbMeta.Spec.Reclaim {
-		reply, err := r.Cred.L4LB().Delete(l4lbMeta.Spec.ID)
-		if err != nil {
-			return ctrl.Result{}, fmt.Errorf("{deleteL4LB} %s", err)
-		}
-		resp, err := http.ParseAPIResponse(reply.Data)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-		if resp.Meta.StatusCode != 200 && !resp.IsSuccess {
-			fmt.Printf("{deleteL4LB} %s\n", resp.Message)
-		}
-	}
 	return r.deleteCRs(l4lb, l4lbMeta)
 }
 
@@ -194,9 +183,11 @@ func (r *L4LBReconciler) deleteCRs(l4lb *k8sv1alpha1.L4LB, l4lbMeta *k8sv1alpha1
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("{deleteCRs} %s", err)
 		}
+	} else {
+		return r.deleteL4LBCR(l4lb)
 	}
 
-	return r.deleteL4LBCR(l4lb)
+	return ctrl.Result{RequeueAfter: requeueInterval}, nil
 }
 
 func (r *L4LBReconciler) deleteL4LBCR(l4lb *k8sv1alpha1.L4LB) (ctrl.Result, error) {
@@ -217,7 +208,7 @@ func (r *L4LBReconciler) deleteL4LBMetaCR(l4lbMeta *k8sv1alpha1.L4LBMeta) (ctrl.
 	if err := r.Delete(ctx, l4lbMeta.DeepCopyObject(), &client.DeleteOptions{}); err != nil {
 		return ctrl.Result{}, fmt.Errorf("{deleteL4LBMetaCR} %s", err)
 	}
-	return ctrl.Result{}, nil
+	return ctrl.Result{RequeueAfter: requeueInterval}, nil
 }
 
 // SetupWithManager .

controllers/l4lbmeta_controller.go

@@ -69,6 +69,43 @@ func (r *L4LBMetaReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	logger := r.Log.WithValues("name", fmt.Sprintf("%s/%s", req.NamespacedName.Namespace, l4lbMeta.Spec.L4LBName))
 	debugLogger = logger.V(int(zapcore.WarnLevel))
 
+	if l4lbMeta.DeletionTimestamp != nil {
+		if l4lbMeta.Spec.ID > 0 && !l4lbMeta.Spec.Reclaim {
+			reply, err := r.Cred.L4LB().Delete(l4lbMeta.Spec.ID)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("{deleteL4LB} %s", err)
+			}
+			resp, err := http.ParseAPIResponse(reply.Data)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			if !resp.IsSuccess && resp.Message != "Invalid load balancer" {
+				return ctrl.Result{}, fmt.Errorf(resp.Message)
+			}
+		}
+
+		l4lbMeta.SetFinalizers(nil)
+		l4lbCtx, l4lbCancel := context.WithTimeout(cntxt, contextTimeout)
+		defer l4lbCancel()
+		err := r.Update(l4lbCtx, l4lbMeta.DeepCopyObject(), &client.UpdateOptions{})
+		if client.IgnoreNotFound(err) != nil {
+			return ctrl.Result{RequeueAfter: requeueInterval}, fmt.Errorf("{DeleteL4LBMetaCR Finalizer} %s", err)
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	if l4lbMeta.GetFinalizers() == nil {
+		l4lbMeta.SetFinalizers([]string{"resource.k8s.netris.ai/delete"})
+		l4lbCtx, l4lbCancel := context.WithTimeout(cntxt, contextTimeout)
+		defer l4lbCancel()
+		err := r.Patch(l4lbCtx, l4lbMeta.DeepCopyObject(), client.Merge, &client.PatchOptions{})
+		if err != nil {
+			logger.Error(fmt.Errorf("{Patch L4LBMeta Finalizer} %s", err), "")
+			return ctrl.Result{RequeueAfter: requeueInterval}, nil
+		}
+	}
+
 	u := uniReconciler{
 		Client:      r.Client,
 		Logger:      logger,
@@ -91,10 +128,6 @@ func (r *L4LBMetaReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		return ctrl.Result{}, err
 	}
 
-	if l4lbMeta.DeletionTimestamp != nil {
-		return ctrl.Result{}, nil
-	}
-
 	if l4lbMeta.Spec.ID == 0 {
 		debugLogger.Info("ID Not found in meta")
 		if l4lbMeta.Spec.Imported {

lbwatcher/lb_watcher.go

@@ -205,7 +205,7 @@ func deleteL4LBs(cl client.Client, lbs []k8sv1alpha1.L4LB) []error {
 func deleteL4LB(cl client.Client, lb k8sv1alpha1.L4LB) error {
 	ctx, cancel := context.WithTimeout(cntxt, contextTimeout)
 	defer cancel()
-	return cl.Delete(ctx, lb.DeepCopyObject(), &client.DeleteOptions{})
+	return client.IgnoreNotFound(cl.Delete(ctx, lb.DeepCopyObject(), &client.DeleteOptions{}))
 }
 
 func updateL4LBs(cl client.Client, lbs []k8sv1alpha1.L4LB, ipAuto map[string]string) []error {
@@ -387,7 +387,9 @@ func (w *Watcher) generateLoadBalancers(clientset *kubernetes.Clientset, autoIPs
 			}
 
 			for _, pod := range podList.Items {
-				hostIPs[pod.Status.HostIP] = 1
+				if pod.Status.HostIP != "" {
+					hostIPs[pod.Status.HostIP] = 1
+				}
 			}
 
 			var lbIPs []lbIP

lbwatcher/services.go

@@ -29,6 +29,7 @@ import (
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/tools/reference"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func getServices(clientset *kubernetes.Clientset, namespace string) (*v1.ServiceList, error) {
@@ -87,12 +88,18 @@ func createEvent(clientset *kubernetes.Clientset, recorder record.EventRecorder,
 	defer cancel()
 	service, err := clientset.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
-		return fmt.Errorf("{createEvent} %s", err)
+		if client.IgnoreNotFound(err) != nil {
+			return fmt.Errorf("{createEvent GetService} %s", err)
+		}
+		return nil
 	}
 
 	ref, err := reference.GetReference(scheme.Scheme, service)
 	if err != nil {
-		return fmt.Errorf("{createEvent} %s", err)
+		if client.IgnoreNotFound(err) != nil {
+			return fmt.Errorf("{createEvent GetReference} %s", err)
+		}
+		return nil
 	}
 	recorder.Event(ref, v1.EventTypeWarning, reason, message)