vpc access support (#387)

* fix(): Fix app pod reconciliation * fix(): Modified logs and loglevel Signed-off-by: Bharath Horatti <[email protected]> * fix(): Updated monitoring pkg version Signed-off-by: Bharath Horatti <[email protected]> * fix(): watch for app pod updates in slice reconciler * watch for slice updates in serviceexport reconciler Signed-off-by: Mridul Gain <[email protected]> * api changes to support vpc service import * gateway types update * service export should be functional from vpc access ns Signed-off-by: Mridul Gain <[email protected]> * exclude workloads from slice overlay network with "kubeslice.io/exclude=true" label Signed-off-by: Mridul Gain <[email protected]> * update controller-gen & unit test fix Signed-off-by: Mridul Gain <[email protected]> --------- Signed-off-by: Bharath Horatti <[email protected]> Signed-off-by: Mridul Gain <[email protected]> Co-authored-by: Bharath Horatti <[email protected]>
kubeslice · Jun 20, 2024 · cb0480a · cb0480a
1 parent 74d07d1
commit cb0480a
Show file tree

Hide file tree

Showing 22 changed files with 513 additions and 247 deletions.
diff --git a/api/v1beta1/slice_types.go b/api/v1beta1/slice_types.go
@@ -91,10 +91,11 @@ type NamespaceIsolationProfile struct {
 
 // ExternalGatewayConfig determines istio ingress/egress configuration
 type ExternalGatewayConfig struct {
-	Ingress     *ExternalGatewayConfigOptions `json:"ingress,omitempty"`
-	Egress      *ExternalGatewayConfigOptions `json:"egress,omitempty"`
-	NsIngress   *ExternalGatewayConfigOptions `json:"nsIngress,omitempty"`
-	GatewayType string                        `json:"gatewayType,omitempty"`
+	Ingress          *ExternalGatewayConfigOptions    `json:"ingress,omitempty"`
+	Egress           *ExternalGatewayConfigOptions    `json:"egress,omitempty"`
+	NsIngress        *ExternalGatewayConfigOptions    `json:"nsIngress,omitempty"`
+	GatewayType      controllerv1alpha1.GatewayType   `json:"gatewayType,omitempty"`
+	VPCServiceAccess controllerv1alpha1.ServiceAccess `json:"vpcServiceAccess,omitempty"`
 }
 
 type ExternalGatewayConfigOptions struct {

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/networking.kubeslice.io_serviceexports.yaml b/config/crd/bases/networking.kubeslice.io_serviceexports.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: serviceexports.networking.kubeslice.io
 spec:
   group: networking.kubeslice.io

diff --git a/config/crd/bases/networking.kubeslice.io_serviceimports.yaml b/config/crd/bases/networking.kubeslice.io_serviceimports.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: serviceimports.networking.kubeslice.io
 spec:
   group: networking.kubeslice.io

diff --git a/config/crd/bases/networking.kubeslice.io_slicegateways.yaml b/config/crd/bases/networking.kubeslice.io_slicegateways.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: slicegateways.networking.kubeslice.io
 spec:
   group: networking.kubeslice.io

diff --git a/config/crd/bases/networking.kubeslice.io_slices.yaml b/config/crd/bases/networking.kubeslice.io_slices.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
+    controller-gen.kubebuilder.io/version: v0.15.0
   name: slices.networking.kubeslice.io
 spec:
   group: networking.kubeslice.io
@@ -107,6 +107,10 @@ spec:
                             type: boolean
                         type: object
                       gatewayType:
+                        enum:
+                        - none
+                        - istio
+                        - envoy
                         type: string
                       ingress:
                         properties:
@@ -118,6 +122,19 @@ spec:
                           enabled:
                             type: boolean
                         type: object
+                      vpcServiceAccess:
+                        properties:
+                          egress:
+                            properties:
+                              enabled:
+                                type: boolean
+                            type: object
+                          ingress:
+                            properties:
+                              enabled:
+                                type: boolean
+                            type: object
+                        type: object
                     type: object
                   namespaceIsolationProfile:
                     description: Namespace Isolation profile contains fields related

diff --git a/controllers/serviceexport/reconciler.go b/controllers/serviceexport/reconciler.go
@@ -32,20 +32,17 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"github.com/kubeslice/kubeslice-monitoring/pkg/metrics"
 	kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1"
 	"github.com/kubeslice/worker-operator/controllers"
+	sliceController "github.com/kubeslice/worker-operator/controllers/slice"
 	ossEvents "github.com/kubeslice/worker-operator/events"
 	"github.com/kubeslice/worker-operator/pkg/logger"
 	"github.com/kubeslice/worker-operator/pkg/utils"
@@ -133,8 +130,8 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 		return ctrl.Result{RequeueAfter: controllers.ReconcileInterval}, nil
 	}
 
-	if !arrayContainsString(slice.Status.ApplicationNamespaces, serviceexport.Namespace) {
-		log.Error(fmt.Errorf("Serviceexport ns is not part of the slice"), "Couldn't onboard serviceexport")
+	if !isValidNameSpace(serviceexport.Namespace, slice) {
+		log.Error(fmt.Errorf("serviceexport ns is not part of the slice"), "couldn't onboard serviceexport")
 		if serviceexport.Status.ExportStatus != kubeslicev1beta1.ExportStatusPending {
 			serviceexport.Status.ExportStatus = kubeslicev1beta1.ExportStatusPending
 			if err := r.Status().Update(ctx, serviceexport); err != nil {
@@ -156,25 +153,22 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 			return ctrl.Result{}, err
 		}
 
-		log.Info("serviceexport updated with ports")
-
 		return ctrl.Result{Requeue: true}, nil
 	}
 
 	res, err, requeue := r.ReconcileAppPod(ctx, serviceexport)
 	if requeue {
-		log.Info("app pods reconciled")
 		debugLog.Info("requeuing after app pod reconcile", "res", res, "er", err)
 		return res, err
 	}
 	r.gaugeEndpoints.WithLabelValues(serviceexport.Spec.Slice, serviceexport.Namespace, serviceexport.Name).Set(float64(serviceexport.Status.AvailableEndpoints))
+
 	res, err, requeue = r.ReconcileIngressGwPod(ctx, serviceexport)
 	if err != nil {
 		utils.RecordEvent(ctx, r.EventRecorder, serviceexport, nil, ossEvents.EventIngressGWPodReconcileFailed, controllerName)
 		return ctrl.Result{}, err
 	}
 	if requeue {
-		log.Info("ingress gw pod reconciled")
 		utils.RecordEvent(ctx, r.EventRecorder, serviceexport, nil, ossEvents.EventIngressGWPodReconciledSuccessfully, controllerName)
 		debugLog.Info("requeuing after ingress gw pod reconcile", "res", res, "er", err)
 		return res, nil
@@ -185,7 +179,6 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 		return ctrl.Result{}, err
 	}
 	if requeue {
-		log.Info("aliases reconciled")
 		r.gaugeAliases.WithLabelValues(serviceexport.Spec.Slice, serviceexport.Namespace, serviceexport.Name).Set(float64(len(serviceexport.Status.Aliases)))
 		debugLog.Info("requeuing after aliases reconcile", "res", res, "er", err)
 		return res, nil
@@ -196,14 +189,12 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 		return ctrl.Result{}, err
 	}
 	if requeue {
-		log.Info("synched serviceexport status")
 		debugLog.Info("requeuing after serviceexport sync", "res", res, "er", err)
 		return res, nil
 	}
 
 	res, err, requeue = r.ReconcileIstio(ctx, serviceexport)
 	if requeue {
-		log.Info("istio reconciled")
 		debugLog.Info("requeuing after Istio reconcile", "res", res, "er", err)
 		return res, err
 	}
@@ -223,6 +214,13 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul
 	}, nil
 }
 
+func isValidNameSpace(ns string, slice *kubeslicev1beta1.Slice) bool {
+	if ns == fmt.Sprintf(sliceController.VPC_NS_FMT, slice.Name) {
+		return true
+	}
+	return arrayContainsString(slice.Status.ApplicationNamespaces, ns)
+}
+
 // Setup ServiceExport Reconciler
 // Initializes metrics and sets up with manager
 func (r *Reconciler) Setup(mgr ctrl.Manager, mf metrics.MetricsFactory) error {
@@ -286,33 +284,36 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&kubeslicev1beta1.ServiceExport{}).
 		Watches(
-			&corev1.Pod{},
-			handler.EnqueueRequestsFromMapFunc(r.mapPodsToServiceExport),
-			builder.WithPredicates(predicate.Funcs{
-				CreateFunc: func(e event.CreateEvent) bool {
-					return false
-				},
-				DeleteFunc: func(e event.DeleteEvent) bool {
-					selector, err := metav1.LabelSelectorAsSelector(&labelSelector)
-					if err != nil {
-						return false
-					}
-					if selector.Matches(labels.Set(e.Object.GetLabels())) {
-						return true
-					}
-					return false
-				},
-				UpdateFunc: func(e event.UpdateEvent) bool {
-					return false
-				},
-				GenericFunc: func(e event.GenericEvent) bool {
-					return false
-				},
-			}),
+			&kubeslicev1beta1.Slice{},
+			handler.EnqueueRequestsFromMapFunc(r.mapServiceExportsToSlice),
 		).
 		Complete(r)
 }
 
+// enqueue requests with service exports belonging under given slice
+func (r *Reconciler) mapServiceExportsToSlice(ctx context.Context, obj client.Object) (recs []reconcile.Request) {
+	log := logger.FromContext(ctx)
+	debugLog := log.V(1)
+	_, ok := obj.(*kubeslicev1beta1.Slice)
+	if !ok {
+		debugLog.Info("Unexpected object type, expected *kubeslicev1beta1.Slice found ", "type", reflect.TypeOf(obj))
+		return
+	}
+	svcexportList := &kubeslicev1beta1.ServiceExportList{}
+	labelSelector := client.MatchingLabels{controllers.ApplicationNamespaceSelectorLabelKey: obj.(*kubeslicev1beta1.Slice).Name}
+	err := r.List(ctx, svcexportList, labelSelector)
+	if err != nil {
+		return
+	}
+	for _, svcexport := range svcexportList.Items {
+		recs = append(recs, reconcile.Request{NamespacedName: types.NamespacedName{
+			Name:      svcexport.Name,
+			Namespace: svcexport.Namespace,
+		}})
+	}
+	return
+}
+
 func (r *Reconciler) GetServiceExport(ctx context.Context, req ctrl.Request, log *logr.Logger) (*kubeslicev1beta1.ServiceExport, error) {
 	serviceexport := &kubeslicev1beta1.ServiceExport{}
 	err := r.Get(ctx, req.NamespacedName, serviceexport)

diff --git a/controllers/serviceexport/serviceexport.go b/controllers/serviceexport/serviceexport.go
@@ -55,14 +55,13 @@ func (r *Reconciler) ReconcileAppPod(
 	serviceexport.Status.ExportStatus = kubeslicev1beta1.ExportStatusPending // Set status to pending
 	serviceexport.Status.AvailableEndpoints = len(appPods)
 
-	log.Info("updating service app pods")
 	debugLog.Info("updating service app pods", "app pods", appPods)
 	err = r.Status().Update(ctx, serviceexport)
 	if err != nil {
 		log.Error(err, "Failed to update ServiceExport status for app pods")
 		return ctrl.Result{}, err, true
 	}
-	log.Info("Service App pod status updated")
+	log.Info("Service App pod status updated", "AvailableEndpoints", len(appPods))
 	return ctrl.Result{Requeue: true}, nil, true
 }
 
@@ -81,7 +80,7 @@ func (r *Reconciler) getAppPods(ctx context.Context, serviceexport *kubeslicev1b
 		return nil, err
 	}
 
-	debugLog.Info("pods matching labels", "count", len(podList.Items))
+	debugLog.Info("pods matching labels", "ServiceExport", serviceexport.Name, "count", len(podList.Items))
 
 	appPods := []kubeslicev1beta1.ServicePod{}
 	appPodsInSlice, err := getAppPodsInSlice(ctx, r.Client, serviceexport.Spec.Slice)
@@ -90,13 +89,14 @@ func (r *Reconciler) getAppPods(ctx context.Context, serviceexport *kubeslicev1b
 		return nil, err
 	}
 
-	debugLog.Info("app pods in slice", "pods", appPodsInSlice)
+	debugLog.Info("app pods in slice", "ServiceExport", serviceexport.Name, "pods", appPodsInSlice)
 	for _, pod := range podList.Items {
 		if pod.Status.Phase == corev1.PodRunning {
 			dnsName := pod.Name + "." + getClusterName() + "." + serviceexport.Name + "." + serviceexport.Namespace + ".svc.slice.local"
 			ip := getNsmIP(&pod, appPodsInSlice)
 			// Avoid adding pods with no nsmip (not part of slice yet)
 			if ip == "" {
+				debugLog.Info("No NSM IP. skipping endpoint", "ServiceExport", serviceexport.Name, pod.Name)
 				continue
 			}
 			appPods = append(appPods, kubeslicev1beta1.ServicePod{