From cb0480a4bebffde463a13def7bd2474a57191477 Mon Sep 17 00:00:00 2001 From: Mridul Gain Date: Thu, 20 Jun 2024 16:44:12 +0530 Subject: [PATCH] vpc access support (#387) * fix(): Fix app pod reconciliation * fix(): Modified logs and loglevel Signed-off-by: Bharath Horatti * fix(): Updated monitoring pkg version Signed-off-by: Bharath Horatti * fix(): watch for app pod updates in slice reconciler * watch for slice updates in serviceexport reconciler Signed-off-by: Mridul Gain * api changes to support vpc service import * gateway types update * service export should be functional from vpc access ns Signed-off-by: Mridul Gain * exclude workloads from slice overlay network with "kubeslice.io/exclude=true" label Signed-off-by: Mridul Gain * update controller-gen & unit test fix Signed-off-by: Mridul Gain --------- Signed-off-by: Bharath Horatti Signed-off-by: Mridul Gain Co-authored-by: Bharath Horatti --- api/v1beta1/slice_types.go | 9 +- api/v1beta1/zz_generated.deepcopy.go | 1 + ...etworking.kubeslice.io_serviceexports.yaml | 2 +- ...etworking.kubeslice.io_serviceimports.yaml | 2 +- ...networking.kubeslice.io_slicegateways.yaml | 2 +- .../bases/networking.kubeslice.io_slices.yaml | 19 +- controllers/serviceexport/reconciler.go | 73 ++--- controllers/serviceexport/serviceexport.go | 8 +- controllers/slice/app_pod.go | 178 ++++++----- controllers/slice/consts.go | 2 + controllers/slice/reconciler.go | 291 +++++++++++++----- go.mod | 11 +- go.sum | 4 +- pkg/hub/controllers/slice_controller.go | 6 +- pkg/webhook/pod/webhook.go | 35 ++- .../controller.kubeslice.io_sliceconfigs.yaml | 14 + .../crd/networking.kubeslice.io_slices.yaml | 44 ++- ...orker.kubeslice.io_workersliceconfigs.yaml | 14 + tests/spoke/slicegw_controller_test.go | 7 +- .../controller/v1alpha1/sliceconfig_types.go | 26 +- .../v1alpha1/workersliceconfig_types.go | 10 +- vendor/modules.txt | 2 +- 22 files changed, 513 insertions(+), 247 deletions(-) diff --git a/api/v1beta1/slice_types.go b/api/v1beta1/slice_types.go index d45c9f6b8..846e0cd7a 100644 --- a/api/v1beta1/slice_types.go +++ b/api/v1beta1/slice_types.go @@ -91,10 +91,11 @@ type NamespaceIsolationProfile struct { // ExternalGatewayConfig determines istio ingress/egress configuration type ExternalGatewayConfig struct { - Ingress *ExternalGatewayConfigOptions `json:"ingress,omitempty"` - Egress *ExternalGatewayConfigOptions `json:"egress,omitempty"` - NsIngress *ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` - GatewayType string `json:"gatewayType,omitempty"` + Ingress *ExternalGatewayConfigOptions `json:"ingress,omitempty"` + Egress *ExternalGatewayConfigOptions `json:"egress,omitempty"` + NsIngress *ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` + GatewayType controllerv1alpha1.GatewayType `json:"gatewayType,omitempty"` + VPCServiceAccess controllerv1alpha1.ServiceAccess `json:"vpcServiceAccess,omitempty"` } type ExternalGatewayConfigOptions struct { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 59d12142b..b52887a92 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -58,6 +58,7 @@ func (in *ExternalGatewayConfig) DeepCopyInto(out *ExternalGatewayConfig) { *out = new(ExternalGatewayConfigOptions) **out = **in } + out.VPCServiceAccess = in.VPCServiceAccess } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalGatewayConfig. diff --git a/config/crd/bases/networking.kubeslice.io_serviceexports.yaml b/config/crd/bases/networking.kubeslice.io_serviceexports.yaml index 98220b823..6b4e0c543 100644 --- a/config/crd/bases/networking.kubeslice.io_serviceexports.yaml +++ b/config/crd/bases/networking.kubeslice.io_serviceexports.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: serviceexports.networking.kubeslice.io spec: group: networking.kubeslice.io diff --git a/config/crd/bases/networking.kubeslice.io_serviceimports.yaml b/config/crd/bases/networking.kubeslice.io_serviceimports.yaml index 239114d0d..d7523d636 100644 --- a/config/crd/bases/networking.kubeslice.io_serviceimports.yaml +++ b/config/crd/bases/networking.kubeslice.io_serviceimports.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: serviceimports.networking.kubeslice.io spec: group: networking.kubeslice.io diff --git a/config/crd/bases/networking.kubeslice.io_slicegateways.yaml b/config/crd/bases/networking.kubeslice.io_slicegateways.yaml index 614d1c509..e05e95083 100644 --- a/config/crd/bases/networking.kubeslice.io_slicegateways.yaml +++ b/config/crd/bases/networking.kubeslice.io_slicegateways.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: slicegateways.networking.kubeslice.io spec: group: networking.kubeslice.io diff --git a/config/crd/bases/networking.kubeslice.io_slices.yaml b/config/crd/bases/networking.kubeslice.io_slices.yaml index 3b7d18992..2dce5ac3e 100644 --- a/config/crd/bases/networking.kubeslice.io_slices.yaml +++ b/config/crd/bases/networking.kubeslice.io_slices.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: slices.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -107,6 +107,10 @@ spec: type: boolean type: object gatewayType: + enum: + - none + - istio + - envoy type: string ingress: properties: @@ -118,6 +122,19 @@ spec: enabled: type: boolean type: object + vpcServiceAccess: + properties: + egress: + properties: + enabled: + type: boolean + type: object + ingress: + properties: + enabled: + type: boolean + type: object + type: object type: object namespaceIsolationProfile: description: Namespace Isolation profile contains fields related diff --git a/controllers/serviceexport/reconciler.go b/controllers/serviceexport/reconciler.go index 7794a29d6..c4d17a331 100644 --- a/controllers/serviceexport/reconciler.go +++ b/controllers/serviceexport/reconciler.go @@ -32,20 +32,17 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/kubeslice/kubeslice-monitoring/pkg/events" "github.com/kubeslice/kubeslice-monitoring/pkg/metrics" kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/controllers" + sliceController "github.com/kubeslice/worker-operator/controllers/slice" ossEvents "github.com/kubeslice/worker-operator/events" "github.com/kubeslice/worker-operator/pkg/logger" "github.com/kubeslice/worker-operator/pkg/utils" @@ -133,8 +130,8 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul return ctrl.Result{RequeueAfter: controllers.ReconcileInterval}, nil } - if !arrayContainsString(slice.Status.ApplicationNamespaces, serviceexport.Namespace) { - log.Error(fmt.Errorf("Serviceexport ns is not part of the slice"), "Couldn't onboard serviceexport") + if !isValidNameSpace(serviceexport.Namespace, slice) { + log.Error(fmt.Errorf("serviceexport ns is not part of the slice"), "couldn't onboard serviceexport") if serviceexport.Status.ExportStatus != kubeslicev1beta1.ExportStatusPending { serviceexport.Status.ExportStatus = kubeslicev1beta1.ExportStatusPending if err := r.Status().Update(ctx, serviceexport); err != nil { @@ -156,25 +153,22 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul return ctrl.Result{}, err } - log.Info("serviceexport updated with ports") - return ctrl.Result{Requeue: true}, nil } res, err, requeue := r.ReconcileAppPod(ctx, serviceexport) if requeue { - log.Info("app pods reconciled") debugLog.Info("requeuing after app pod reconcile", "res", res, "er", err) return res, err } r.gaugeEndpoints.WithLabelValues(serviceexport.Spec.Slice, serviceexport.Namespace, serviceexport.Name).Set(float64(serviceexport.Status.AvailableEndpoints)) + res, err, requeue = r.ReconcileIngressGwPod(ctx, serviceexport) if err != nil { utils.RecordEvent(ctx, r.EventRecorder, serviceexport, nil, ossEvents.EventIngressGWPodReconcileFailed, controllerName) return ctrl.Result{}, err } if requeue { - log.Info("ingress gw pod reconciled") utils.RecordEvent(ctx, r.EventRecorder, serviceexport, nil, ossEvents.EventIngressGWPodReconciledSuccessfully, controllerName) debugLog.Info("requeuing after ingress gw pod reconcile", "res", res, "er", err) return res, nil @@ -185,7 +179,6 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul return ctrl.Result{}, err } if requeue { - log.Info("aliases reconciled") r.gaugeAliases.WithLabelValues(serviceexport.Spec.Slice, serviceexport.Namespace, serviceexport.Name).Set(float64(len(serviceexport.Status.Aliases))) debugLog.Info("requeuing after aliases reconcile", "res", res, "er", err) return res, nil @@ -196,14 +189,12 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul return ctrl.Result{}, err } if requeue { - log.Info("synched serviceexport status") debugLog.Info("requeuing after serviceexport sync", "res", res, "er", err) return res, nil } res, err, requeue = r.ReconcileIstio(ctx, serviceexport) if requeue { - log.Info("istio reconciled") debugLog.Info("requeuing after Istio reconcile", "res", res, "er", err) return res, err } @@ -223,6 +214,13 @@ func (r Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resul }, nil } +func isValidNameSpace(ns string, slice *kubeslicev1beta1.Slice) bool { + if ns == fmt.Sprintf(sliceController.VPC_NS_FMT, slice.Name) { + return true + } + return arrayContainsString(slice.Status.ApplicationNamespaces, ns) +} + // Setup ServiceExport Reconciler // Initializes metrics and sets up with manager func (r *Reconciler) Setup(mgr ctrl.Manager, mf metrics.MetricsFactory) error { @@ -286,33 +284,36 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&kubeslicev1beta1.ServiceExport{}). Watches( - &corev1.Pod{}, - handler.EnqueueRequestsFromMapFunc(r.mapPodsToServiceExport), - builder.WithPredicates(predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return false - }, - DeleteFunc: func(e event.DeleteEvent) bool { - selector, err := metav1.LabelSelectorAsSelector(&labelSelector) - if err != nil { - return false - } - if selector.Matches(labels.Set(e.Object.GetLabels())) { - return true - } - return false - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return false - }, - GenericFunc: func(e event.GenericEvent) bool { - return false - }, - }), + &kubeslicev1beta1.Slice{}, + handler.EnqueueRequestsFromMapFunc(r.mapServiceExportsToSlice), ). Complete(r) } +// enqueue requests with service exports belonging under given slice +func (r *Reconciler) mapServiceExportsToSlice(ctx context.Context, obj client.Object) (recs []reconcile.Request) { + log := logger.FromContext(ctx) + debugLog := log.V(1) + _, ok := obj.(*kubeslicev1beta1.Slice) + if !ok { + debugLog.Info("Unexpected object type, expected *kubeslicev1beta1.Slice found ", "type", reflect.TypeOf(obj)) + return + } + svcexportList := &kubeslicev1beta1.ServiceExportList{} + labelSelector := client.MatchingLabels{controllers.ApplicationNamespaceSelectorLabelKey: obj.(*kubeslicev1beta1.Slice).Name} + err := r.List(ctx, svcexportList, labelSelector) + if err != nil { + return + } + for _, svcexport := range svcexportList.Items { + recs = append(recs, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: svcexport.Name, + Namespace: svcexport.Namespace, + }}) + } + return +} + func (r *Reconciler) GetServiceExport(ctx context.Context, req ctrl.Request, log *logr.Logger) (*kubeslicev1beta1.ServiceExport, error) { serviceexport := &kubeslicev1beta1.ServiceExport{} err := r.Get(ctx, req.NamespacedName, serviceexport) diff --git a/controllers/serviceexport/serviceexport.go b/controllers/serviceexport/serviceexport.go index ae9c3196a..bd6fc4b3f 100644 --- a/controllers/serviceexport/serviceexport.go +++ b/controllers/serviceexport/serviceexport.go @@ -55,14 +55,13 @@ func (r *Reconciler) ReconcileAppPod( serviceexport.Status.ExportStatus = kubeslicev1beta1.ExportStatusPending // Set status to pending serviceexport.Status.AvailableEndpoints = len(appPods) - log.Info("updating service app pods") debugLog.Info("updating service app pods", "app pods", appPods) err = r.Status().Update(ctx, serviceexport) if err != nil { log.Error(err, "Failed to update ServiceExport status for app pods") return ctrl.Result{}, err, true } - log.Info("Service App pod status updated") + log.Info("Service App pod status updated", "AvailableEndpoints", len(appPods)) return ctrl.Result{Requeue: true}, nil, true } @@ -81,7 +80,7 @@ func (r *Reconciler) getAppPods(ctx context.Context, serviceexport *kubeslicev1b return nil, err } - debugLog.Info("pods matching labels", "count", len(podList.Items)) + debugLog.Info("pods matching labels", "ServiceExport", serviceexport.Name, "count", len(podList.Items)) appPods := []kubeslicev1beta1.ServicePod{} appPodsInSlice, err := getAppPodsInSlice(ctx, r.Client, serviceexport.Spec.Slice) @@ -90,13 +89,14 @@ func (r *Reconciler) getAppPods(ctx context.Context, serviceexport *kubeslicev1b return nil, err } - debugLog.Info("app pods in slice", "pods", appPodsInSlice) + debugLog.Info("app pods in slice", "ServiceExport", serviceexport.Name, "pods", appPodsInSlice) for _, pod := range podList.Items { if pod.Status.Phase == corev1.PodRunning { dnsName := pod.Name + "." + getClusterName() + "." + serviceexport.Name + "." + serviceexport.Namespace + ".svc.slice.local" ip := getNsmIP(&pod, appPodsInSlice) // Avoid adding pods with no nsmip (not part of slice yet) if ip == "" { + debugLog.Info("No NSM IP. skipping endpoint", "ServiceExport", serviceexport.Name, pod.Name) continue } appPods = append(appPods, kubeslicev1beta1.ServicePod{ diff --git a/controllers/slice/app_pod.go b/controllers/slice/app_pod.go index ddd5b6668..1b142406a 100644 --- a/controllers/slice/app_pod.go +++ b/controllers/slice/app_pod.go @@ -24,9 +24,7 @@ import ( kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/controllers" - ossEvents "github.com/kubeslice/worker-operator/events" "github.com/kubeslice/worker-operator/pkg/logger" - "github.com/kubeslice/worker-operator/pkg/utils" webhook "github.com/kubeslice/worker-operator/pkg/webhook/pod" corev1 "k8s.io/api/core/v1" @@ -34,23 +32,43 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +func isAppPodStatusChanged(current []kubeslicev1beta1.AppPod, old []kubeslicev1beta1.AppPod) bool { + if len(current) != len(old) { + return true + } + + oldPodMap := make(map[string]string) + + for _, op := range old { + oldPodMap[op.PodName] = op.PodIP + } + + for _, cp := range current { + if _, ok := oldPodMap[cp.PodName]; !ok { + return true + } + } + + return false +} + func (r *SliceReconciler) getAppPods(ctx context.Context, slice *kubeslicev1beta1.Slice) ([]kubeslicev1beta1.AppPod, error) { log := logger.FromContext(ctx).WithValues("type", "app_pod") debugLog := log.V(1) podList := &corev1.PodList{} listOpts := []client.ListOption{ - client.MatchingLabels(labelsForAppPods()), + client.MatchingLabels(labelsForAppPods(slice.Name)), } if err := r.List(ctx, podList, listOpts...); err != nil { log.Error(err, "Failed to list pods") return nil, err } + appPods := []kubeslicev1beta1.AppPod{} - for _, pod := range podList.Items { + for _, pod := range podList.Items { a := pod.Annotations - if !isAppPodConnectedToSliceRouter(a, "vl3-service-"+slice.Name) { // Could get noisy. Review needed. debugLog.Info("App pod is not part of the slice", "pod", pod.Name, "slice", slice.Name) @@ -65,12 +83,13 @@ func (r *SliceReconciler) getAppPods(ctx context.Context, slice *kubeslicev1beta }) } } + return appPods, nil } // labelsForAppPods returns the labels for App pods -func labelsForAppPods() map[string]string { - return map[string]string{webhook.PodInjectLabelKey: "app"} +func labelsForAppPods(sliceName string) map[string]string { + return map[string]string{webhook.PodInjectLabelKey: "app", ApplicationNamespaceSelectorLabelKey: sliceName} } func isAppPodConnectedToSliceRouter(annotations map[string]string, sliceRouter string) bool { @@ -93,14 +112,23 @@ func (r *SliceReconciler) ReconcileAppPod(ctx context.Context, slice *kubeslicev return ctrl.Result{}, err, true } debugLog.Info("Got pods connected to slice", "result", podsConnectedToSlice) - corePodList := &corev1.PodList{} - listOpts := []client.ListOption{ - client.MatchingLabels(labelsForAppPods()), - } - if err := r.List(ctx, corePodList, listOpts...); err != nil { - log.Error(err, "Failed to list pods") + + appPods, err := r.getAppPods(ctx, slice) + debugLog.Info("app pods", "slice", sliceName, "pods", appPods, "err", err) + if err != nil { return ctrl.Result{}, err, true } + + updateNeeded := false + + // First, reconcile the current set of pods in the slice namespaces with the pods stored + // in the slice status. + if isAppPodStatusChanged(appPods, slice.Status.AppPods) { + log.Info("Latest app pods do not match the pods in slice status. Update needed", "slice", sliceName) + updateNeeded = true + slice.Status.AppPods = appPods + } + for i := range slice.Status.AppPods { pod := &slice.Status.AppPods[i] debugLog.Info("getting app pod connectivity status", "podIp", pod.PodIP, "podName", pod.PodName) @@ -108,25 +136,37 @@ func (r *SliceReconciler) ReconcileAppPod(ctx context.Context, slice *kubeslicev // Presence of an nsm interface is good enough for now to consider the app pod as healthy with // respect to its connectivity to the slice. if appPodConnectedToSlice == nil { - debugLog.Info("App pod unhealthy: Not connected to slice", "podName", pod.PodName) - - if pod.NsmIP != "" || pod.NsmPeerIP != "" { - return r.updateSliceAppPodStatus(ctx, pod, slice) + debugLog.Info("App pod not connected to slice", "podName", pod.PodName) + if pod.NsmIP != "" { + pod.NsmIP = "" + updateNeeded = true } - debugLog.Info("App pod unhealthy, skipping reconciliation") continue } if pod.NsmIP != appPodConnectedToSlice.NsmIP { - pod.NsmIP, pod.NsmPeerIP, pod.NsmInterface = - appPodConnectedToSlice.NsmIP, appPodConnectedToSlice.NsmPeerIP, appPodConnectedToSlice.NsmInterface - slice.Status.AppPodsUpdatedOn = time.Now().Unix() - log.Info("app pod status changed", "nsmIp", pod.NsmIP, "peerIp", pod.NsmPeerIP) + debugLog.Info("app pod status changed", "old nsmIp", pod.NsmIP, "new nsmIp", appPodConnectedToSlice.NsmIP) + pod.NsmIP = appPodConnectedToSlice.NsmIP + updateNeeded = true + } + } - //Label pod with NSM IP - return r.labelAppPodWithNsmIp(ctx, pod, corePodList, slice) + if updateNeeded { + slice.Status.AppPodsUpdatedOn = time.Now().Unix() + err := r.Status().Update(ctx, slice) + if err != nil { + log.Error(err, "Failed to update Slice status for app pods") + return ctrl.Result{}, err, true } } + + // Reconcile nsm IP label on app pods + err = r.labelAppPodWithNsmIp(ctx, slice) + if err != nil { + log.Error(err, "Failed to label app pods") + return ctrl.Result{}, err, true + } + return ctrl.Result{}, nil, false } @@ -144,60 +184,50 @@ func findAppPodConnectedToSlice(podName string, connectedPods []kubeslicev1beta1 return nil } -// findPodInPodList returns the index of the pod in the podList that matches -// the input pod to be found -func findPodInPodList(podName string, podList *corev1.PodList) int { - for i := range podList.Items { - pod := podList.Items[i] - if podName == pod.Name { - return i - } - } - return -1 -} - -func (r *SliceReconciler) updateSliceAppPodStatus(ctx context.Context, pod *kubeslicev1beta1.AppPod, slice *kubeslicev1beta1.Slice) (ctrl.Result, error, bool) { +func (r *SliceReconciler) labelAppPodWithNsmIp(ctx context.Context, slice *kubeslicev1beta1.Slice) error { log := logger.FromContext(ctx).WithValues("type", "app_pod") debugLog := log.V(1) - pod.NsmIP = "" - pod.NsmPeerIP = "" - slice.Status.AppPodsUpdatedOn = time.Now().Unix() - debugLog.Info("Setting app pod nsm and peer Ip to null") - err := r.Status().Update(ctx, slice) - if err != nil { - log.Error(err, "Failed to update Slice status for app pods which sets nsmip and peerip to null") - return ctrl.Result{}, err, true + + corePodList := &corev1.PodList{} + listOpts := []client.ListOption{ + client.MatchingLabels(labelsForAppPods(slice.Name)), + } + if err := r.List(ctx, corePodList, listOpts...); err != nil { + log.Error(err, "Label app pods error: Failed to list pods") + return err } - debugLog.Info("App pod status updated and nsmip peerip set to null") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceUpdated, controllerName) - return ctrl.Result{}, nil, true -} -func (r *SliceReconciler) labelAppPodWithNsmIp(ctx context.Context, pod *kubeslicev1beta1.AppPod, corePodList *corev1.PodList, slice *kubeslicev1beta1.Slice) (ctrl.Result, error, bool) { - log := logger.FromContext(ctx).WithValues("type", "app_pod") - debugLog := log.V(1) - podIndex := findPodInPodList(pod.PodName, corePodList) - if podIndex == -1 { - debugLog.Info("Could not find pod in podList, skipping nsmIP labelling") - } else { - corePod := corePodList.Items[podIndex] - labels := corePod.GetLabels() - labels[controllers.NSMIPLabelSelectorKey] = pod.NsmIP - corePod.SetLabels(labels) - - err := r.Update(ctx, &corePod) - if err != nil { - log.Error(err, "Failed to update NSM IP label for app pod") - return ctrl.Result{}, err, true + for _, pod := range corePodList.Items { + updateNeeded := false + podInSlice := findAppPodConnectedToSlice(pod.Name, slice.Status.AppPods) + if podInSlice == nil { + continue + } + labels := pod.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + _, ok := labels[controllers.NSMIPLabelSelectorKey] + if !ok { + updateNeeded = true + labels[controllers.NSMIPLabelSelectorKey] = podInSlice.NsmIP + } else { + if labels[controllers.NSMIPLabelSelectorKey] != podInSlice.NsmIP { + updateNeeded = true + labels[controllers.NSMIPLabelSelectorKey] = podInSlice.NsmIP + } + } + + if updateNeeded { + pod.SetLabels(labels) + err := r.Update(ctx, &pod) + if err != nil { + log.Error(err, "Failed to update NSM IP label for app pod", "pod", pod.Name) + return err + } + debugLog.Info("App pod label added/updated", "pod", pod.Name, "nsmIP", podInSlice.NsmIP) } - debugLog.Info("App pod label added/updated", "nsmIP", pod.NsmIP) - } - err := r.Status().Update(ctx, slice) - if err != nil { - log.Error(err, "Failed to update Slice status for app pods") - return ctrl.Result{}, err, true } - log.Info("App pod status updated") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceUpdated, controllerName) - return ctrl.Result{}, nil, true + + return nil } diff --git a/controllers/slice/consts.go b/controllers/slice/consts.go index c7e7baea4..142d5407d 100644 --- a/controllers/slice/consts.go +++ b/controllers/slice/consts.go @@ -6,4 +6,6 @@ const ( ApplicationNamespaceSelectorLabelKey = "kubeslice.io/slice" AllowedNamespaceAnnotationKey = "kubeslice.io/trafficAllowedToSlices" InjectSidecarKey = "kubeslice.io/inject" + + VpcEgressGwNsName = "%s-vpc-egress-gw-system" ) diff --git a/controllers/slice/reconciler.go b/controllers/slice/reconciler.go index 738bd1f39..90a1f6c1e 100644 --- a/controllers/slice/reconciler.go +++ b/controllers/slice/reconciler.go @@ -21,35 +21,46 @@ package slice import ( "context" "fmt" - "time" + "log" + "reflect" - "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" retry "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/kubeslice/apis/pkg/controller/v1alpha1" + "github.com/go-logr/logr" + controllerv1alpha1 "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/kubeslice-monitoring/pkg/events" "github.com/kubeslice/kubeslice-monitoring/pkg/metrics" kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/controllers" ossEvents "github.com/kubeslice/worker-operator/events" - "github.com/kubeslice/worker-operator/pkg/logger" "github.com/kubeslice/worker-operator/pkg/manifest" "github.com/kubeslice/worker-operator/pkg/utils" "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -var sliceFinalizer = "networking.kubeslice.io/slice-finalizer" -var controllerName = "sliceReconciler" +const VPC_NS_FMT = "%s-vpc-access-gw-system" + +var ( + sliceFinalizer = "networking.kubeslice.io/slice-finalizer" + controllerName = "sliceReconciler" +) // SliceReconciler reconciles a Slice object type SliceReconciler struct { @@ -151,7 +162,7 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, err } - if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode != v1alpha1.NONET { + if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode != controllerv1alpha1.NONET { if slice.Status.DNSIP == "" { requeue, result, err := r.handleDnsSvc(ctx, slice) if requeue { @@ -166,7 +177,7 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return res, err } - if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode == v1alpha1.NONET { + if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode == controllerv1alpha1.NONET { debugLog.Info("No communication slice, skipping reconciliation of qos, netop, egw, router etc") // to support net to no-net switching write a function to delete network components if present } else { @@ -179,27 +190,40 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } } - appPods, err := r.getAppPods(ctx, slice) - debugLog.Info("app pods", "pods", appPods, "err", err) + if isIngressConfigured(slice) { + debugLog.Info("Installing ingress") + err = manifest.InstallIngress(ctx, r.Client, slice) + if err != nil { + log.Error(err, "unable to install ingress") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceIngressInstallFailed, controllerName) + return ctrl.Result{}, nil + } + } - // expose the number of app pods metric of a slice - r.exposeMetric(appPods, slice) + res, err, requeue = r.ReconcileSliceRouter(ctx, slice) + if err != nil { + log.Error(err, "Failed to reconcile slice router") + } + if requeue { + return res, err + } - if isAppPodStatusChanged(appPods, slice.Status.AppPods) { - log.Info("App pod status changed") - return r.handleAppPodStatusChange(appPods, slice, ctx) + res, err, requeue = r.ReconcileSliceGwEdge(ctx, slice) + if err != nil { + log.Error(err, "Slice Edge reconciliation failed") + return res, err + } + if requeue { + return ctrl.Result{ + Requeue: true, + }, nil } - if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode == v1alpha1.NONET { - debugLog.Info("No communication slice, skipping reconciliation of apppods") - // to support net to no-net switching write a function to remove nsm interfaces, ips, labels with nsmip etc from existing app pods - } else { - debugLog.Info("reconciling app pods") - res, err, requeue = r.ReconcileAppPod(ctx, slice) - if err != nil { - log.Error(err, "App pod reconciliation failed") - return res, err - } + debugLog.Info("reconciling app pods") + res, err, requeue = r.ReconcileAppPod(ctx, slice) + if err != nil { + log.Error(err, "App pod reconciliation failed") + return res, err } if requeue { @@ -215,6 +239,8 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl Requeue: true, }, nil } + // expose the number of app pods metric of a slice + r.exposeMetric(slice.Status.AppPods, slice) return ctrl.Result{ RequeueAfter: controllers.ReconcileInterval, @@ -254,24 +280,27 @@ func (r *SliceReconciler) reconcileNetworkComponents(ctx context.Context, slice return res, nil, true } - debugLog.Info("ExternalGatewayConfig", "egw", slice.Status.SliceConfig) - if isEgressConfigured(slice) { - debugLog.Info("Installing egress") - err = manifest.InstallEgress(ctx, r.Client, slice) - if err != nil { - log.Error(err, "unable to install egress") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceEgressInstallFailed, controllerName) - return ctrl.Result{}, err, false + debugLog.Info("ExternalGatewayConfig", "obj", slice.Status.SliceConfig.ExternalGatewayConfig) + if slice.Status.SliceConfig.ExternalGatewayConfig != nil && + slice.Status.SliceConfig.ExternalGatewayConfig.GatewayType == controllerv1alpha1.GATEWAY_TYPE_ISTIO { + if isEgressConfigured(slice) { + debugLog.Info("Installing istio egress") + err = manifest.InstallEgress(ctx, r.Client, slice) + if err != nil { + log.Error(err, "unable to install egress") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceEgressInstallFailed, controllerName) + return ctrl.Result{}, err, false + } } - } - if isIngressConfigured(slice) { - debugLog.Info("Installing ingress") - err = manifest.InstallIngress(ctx, r.Client, slice) - if err != nil { - log.Error(err, "unable to install ingress") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceIngressInstallFailed, controllerName) - return ctrl.Result{}, err, false + if isIngressConfigured(slice) { + debugLog.Info("Installing istio ingress") + err = manifest.InstallIngress(ctx, r.Client, slice) + if err != nil { + log.Error(err, "unable to install ingress") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceIngressInstallFailed, controllerName) + return ctrl.Result{}, err, false + } } } @@ -297,22 +326,6 @@ func (r *SliceReconciler) exposeMetric(appPods []kubeslicev1beta1.AppPod, slice } } -func (r *SliceReconciler) handleAppPodStatusChange(appPods []kubeslicev1beta1.AppPod, slice *kubeslicev1beta1.Slice, ctx context.Context) (reconcile.Result, error) { - log := logger.FromContext(ctx).WithName("app-pod-update") - - slice.Status.AppPods = appPods - slice.Status.AppPodsUpdatedOn = time.Now().Unix() - err := r.Status().Update(ctx, slice) - if err != nil { - log.Error(err, "Failed to update Slice status for app pods") - return ctrl.Result{}, err - } - log.Info("App pod status updated in slice") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceUpdated, controllerName) - - return ctrl.Result{Requeue: true}, nil -} - func isEgressConfigured(slice *kubeslicev1beta1.Slice) bool { return slice.Status.SliceConfig.ExternalGatewayConfig != nil && slice.Status.SliceConfig.ExternalGatewayConfig.Egress.Enabled } @@ -383,7 +396,7 @@ func (r *SliceReconciler) handleSliceDeletion(slice *kubeslicev1beta1.Slice, ctx if controllerutil.ContainsFinalizer(slice, sliceFinalizer) { log.Info("Deleting slice", "slice", slice.Name) if slice.Status.SliceConfig != nil && - slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode != v1alpha1.NONET { + slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode != controllerv1alpha1.NONET { err := r.SendSliceDeletionEventToNetOp(ctx, req.NamespacedName.Name, req.NamespacedName.Namespace) if err != nil { log.Error(err, "Failed to send slice deletetion event to netop") @@ -405,26 +418,6 @@ func (r *SliceReconciler) handleSliceDeletion(slice *kubeslicev1beta1.Slice, ctx return false, reconcile.Result{}, nil } -func isAppPodStatusChanged(current []kubeslicev1beta1.AppPod, old []kubeslicev1beta1.AppPod) bool { - if len(current) != len(old) { - return true - } - - s := make(map[string]string) - - for _, c := range old { - s[c.PodIP] = c.PodName - } - - for _, c := range current { - if s[c.PodIP] != c.PodName { - return true - } - } - - return false -} - // Setup SliceReconciler // Initializes metrics and sets up with manager func (r *SliceReconciler) Setup(mgr ctrl.Manager, mf metrics.MetricsFactory) error { @@ -437,10 +430,150 @@ func (r *SliceReconciler) Setup(mgr ctrl.Manager, mf metrics.MetricsFactory) err // SetupWithManager sets up the controller with the Manager. func (r *SliceReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Create a label selector that matches based on the existence of a label key + sliceSelector := labels.NewSelector() + requirement, err := labels.NewRequirement(controllers.ApplicationNamespaceSelectorLabelKey, selection.Exists, nil) + if err != nil { + log.Fatalf("Error creating label requirement: %v", err) + } + sliceSelector = sliceSelector.Add(*requirement) return ctrl.NewControllerManagedBy(mgr). For(&kubeslicev1beta1.Slice{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Pod{}). Owns(&kubeslicev1beta1.SliceGateway{}). + Watches( + &appsv1.Deployment{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) (recs []reconcile.Request) { + log := logger.FromContext(ctx) + debuglog := log.V(1) + debuglog.Info("Triggered slice reconciler by", "type", reflect.TypeOf(o)) + sliceName := o.(*appsv1.Deployment).Labels[controllers.ApplicationNamespaceSelectorLabelKey] + recs = append(recs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: sliceName, + Namespace: controllers.ControlPlaneNamespace, + }, + }) + debuglog.Info("Requeuing slice", "name", sliceName) + return + }), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return sliceSelector.Matches(labels.Set(e.Object.GetLabels())) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + if sliceSelector.Matches(labels.Set(e.ObjectOld.GetLabels())) { + oldObj, ok := e.ObjectOld.(*appsv1.Deployment) + if !ok { + return false + } + newObj, ok := e.ObjectNew.(*appsv1.Deployment) + if !ok { + return false + } + // trigger in case of scale down + if oldObj.Status.ReadyReplicas > newObj.Status.ReadyReplicas { + return true + } + } + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). + Watches( + &appsv1.DaemonSet{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) (recs []reconcile.Request) { + log := logger.FromContext(ctx) + debuglog := log.V(1) + debuglog.Info("Triggered slice reconciler by", "type", reflect.TypeOf(o)) + sliceName := o.(*appsv1.DaemonSet).Labels[controllers.ApplicationNamespaceSelectorLabelKey] + recs = append(recs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: sliceName, + Namespace: controllers.ControlPlaneNamespace, + }, + }) + debuglog.Info("Requeuing slice", "name", sliceName) + return + }), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return sliceSelector.Matches(labels.Set(e.Object.GetLabels())) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + if sliceSelector.Matches(labels.Set(e.ObjectOld.GetLabels())) { + oldObj, ok := e.ObjectOld.(*appsv1.DaemonSet) + if !ok { + return false + } + newObj, ok := e.ObjectNew.(*appsv1.DaemonSet) + if !ok { + return false + } + if oldObj.Status.NumberReady > newObj.Status.NumberReady { + return true + } + } + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). + Watches( + &appsv1.StatefulSet{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) (recs []reconcile.Request) { + log := logger.FromContext(ctx) + debuglog := log.V(1) + debuglog.Info("Triggered slice reconciler by", "type", reflect.TypeOf(o)) + sliceName := o.(*appsv1.StatefulSet).Labels[controllers.ApplicationNamespaceSelectorLabelKey] + recs = append(recs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: sliceName, + Namespace: controllers.ControlPlaneNamespace, + }, + }) + debuglog.Info("Requeuing slice", "name", sliceName) + return + }), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return sliceSelector.Matches(labels.Set(e.Object.GetLabels())) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + if sliceSelector.Matches(labels.Set(e.ObjectOld.GetLabels())) { + oldObj, ok := e.ObjectOld.(*appsv1.StatefulSet) + if !ok { + return false + } + newObj, ok := e.ObjectNew.(*appsv1.StatefulSet) + if !ok { + return false + } + if oldObj.Status.ReadyReplicas > newObj.Status.ReadyReplicas { + return true + } + } + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). Complete(r) } diff --git a/go.mod b/go.mod index 9a3004afe..d3bd5e283 100644 --- a/go.mod +++ b/go.mod @@ -2,14 +2,16 @@ module github.com/kubeslice/worker-operator go 1.22.1 +// replace github.com/kubeslice/apis => ../../misc/apis + require ( contrib.go.opencensus.io/exporter/prometheus v0.4.1 github.com/avast/retry-go v3.0.0+incompatible github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 - //github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.6.0 - github.com/kubeslice/apis v0.3.0 + github.com/kubeslice/apis v0.3.1 github.com/kubeslice/gateway-sidecar v0.2.0 github.com/kubeslice/kubeslice-monitoring v0.2.1 github.com/kubeslice/netops v0.1.3 @@ -37,10 +39,7 @@ require ( sigs.k8s.io/controller-runtime v0.16.3 ) -require ( - github.com/golang/protobuf v1.5.3 - sigs.k8s.io/gateway-api v1.0.0 -) +require sigs.k8s.io/gateway-api v1.0.0 require ( github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 9894ca948..a3a8486ca 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeslice/apis v0.3.0 h1:sSaMLKWN9OfUdcTYAcIaUpRwlZqrmgKLPG5I8dYYH2k= -github.com/kubeslice/apis v0.3.0/go.mod h1:YDSfpIsQM+FtQPaZVGNCTZnlp3viWuQhkjJjIHQdaYs= +github.com/kubeslice/apis v0.3.1 h1:5t8vTpyiu+2w8vLq4ATpEfhPJX4gG6mMZF+PcJQw8gM= +github.com/kubeslice/apis v0.3.1/go.mod h1:YDSfpIsQM+FtQPaZVGNCTZnlp3viWuQhkjJjIHQdaYs= github.com/kubeslice/gateway-sidecar v0.2.0 h1:Ja3fIUivuSjUFQ4lPCt79ATq99BxslvAFYUwV9Urpy4= github.com/kubeslice/gateway-sidecar v0.2.0/go.mod h1:nM1+Wjud2vk44cUg+9iwBbWTpqI+2Ecbn9NuaHEs9aY= github.com/kubeslice/kubeslice-monitoring v0.2.1 h1:wtmIEigpQoKzuckof7QRqdsaa4lV/rqxd/FcmOj5N5Q= diff --git a/pkg/hub/controllers/slice_controller.go b/pkg/hub/controllers/slice_controller.go index b7edf9296..c36503330 100644 --- a/pkg/hub/controllers/slice_controller.go +++ b/pkg/hub/controllers/slice_controller.go @@ -28,17 +28,18 @@ import ( spokev1alpha1 "github.com/kubeslice/apis/pkg/worker/v1alpha1" "github.com/kubeslice/kubeslice-monitoring/pkg/events" "github.com/kubeslice/kubeslice-monitoring/pkg/metrics" - ossEvents "github.com/kubeslice/worker-operator/events" - kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" + ossEvents "github.com/kubeslice/worker-operator/events" "github.com/kubeslice/worker-operator/pkg/gwsidecar" "github.com/kubeslice/worker-operator/pkg/logger" "github.com/kubeslice/worker-operator/pkg/utils" "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -315,6 +316,7 @@ func (r *SliceReconciler) updateSliceConfig(ctx context.Context, meshSlice *kube NsIngress: &kubeslicev1beta1.ExternalGatewayConfigOptions{ Enabled: extGwCfg.NsIngress.Enabled, }, + VPCServiceAccess: extGwCfg.VPCServiceAccess, } return r.MeshClient.Status().Update(ctx, meshSlice) diff --git a/pkg/webhook/pod/webhook.go b/pkg/webhook/pod/webhook.go index 390b00ed5..1bc7a1f73 100644 --- a/pkg/webhook/pod/webhook.go +++ b/pkg/webhook/pod/webhook.go @@ -44,6 +44,7 @@ const ( controlPlaneNamespace = "kubeslice-system" nsmInjectAnnotaionKey1 = "ns.networkservicemesh.io" nsmInjectAnnotaionKey2 = "networkservicemesh.io" + kubesliceExcludeKey = "kubeslice.io/exclude" ) var ( @@ -78,11 +79,10 @@ func (wh *WebhookServer) Handle(ctx context.Context, req admission.Request) admi } if mutate, sliceName := wh.MutationRequired(pod.ObjectMeta, ctx, req.Kind.Kind); !mutate { - log.Info("mutation not required for pod", "pod metadata", pod.ObjectMeta) + log.Info("mutation not required for pod", "pod metadata", pod.ObjectMeta.Name) } else { - log.Info("mutating pod", "pod metadata", pod.ObjectMeta) + log.Info("mutating pod", "pod metadata", pod.ObjectMeta.Name) pod = MutatePod(pod, sliceName) - log.Info("mutated pod", "pod metadata", pod.ObjectMeta) } marshaled, err := json.Marshal(pod) @@ -101,7 +101,6 @@ func (wh *WebhookServer) Handle(ctx context.Context, req admission.Request) admi if mutate, sliceName := wh.MutationRequired(deploy.ObjectMeta, ctx, req.Kind.Kind); !mutate { log.Info("mutation not required for deployment", "pod metadata", deploy.Spec.Template.ObjectMeta) } else { - log.Info("mutating deploy", "pod metadata", deploy.Spec.Template.ObjectMeta) deploy = MutateDeployment(deploy, sliceName) log.Info("mutated deploy", "pod metadata", deploy.Spec.Template.ObjectMeta) } @@ -122,7 +121,6 @@ func (wh *WebhookServer) Handle(ctx context.Context, req admission.Request) admi if mutate, sliceName := wh.MutationRequired(statefulset.ObjectMeta, ctx, req.Kind.Kind); !mutate { log.Info("mutation not required for statefulsets", "pod metadata", statefulset.Spec.Template.ObjectMeta) } else { - log.Info("mutating statefulset", "pod metadata", statefulset.Spec.Template.ObjectMeta) statefulset = MutateStatefulset(statefulset, sliceName) log.Info("mutated statefulset", "pod metadata", statefulset.Spec.Template.ObjectMeta) } @@ -143,7 +141,6 @@ func (wh *WebhookServer) Handle(ctx context.Context, req admission.Request) admi if mutate, sliceName := wh.MutationRequired(daemonset.ObjectMeta, ctx, req.Kind.Kind); !mutate { log.Info("mutation not required for daemonset", "pod metadata", daemonset.Spec.Template.ObjectMeta) } else { - log.Info("mutating daemonset", "pod metadata", daemonset.Spec.Template.ObjectMeta) daemonset = MutateDaemonSet(daemonset, sliceName) log.Info("mutated daemonset", "pod metadata", daemonset.Spec.Template.ObjectMeta) } @@ -226,6 +223,11 @@ func MutateDeployment(deploy *appsv1.Deployment, sliceName string) *appsv1.Deplo labels[PodInjectLabelKey] = "app" labels[admissionWebhookAnnotationInjectKey] = sliceName + if deploy.ObjectMeta.Labels == nil { + deploy.ObjectMeta.Labels = make(map[string]string) + } + deploy.ObjectMeta.Labels[admissionWebhookAnnotationInjectKey] = sliceName + return deploy } @@ -250,6 +252,11 @@ func MutateStatefulset(ss *appsv1.StatefulSet, sliceName string) *appsv1.Statefu labels[PodInjectLabelKey] = "app" labels[admissionWebhookAnnotationInjectKey] = sliceName + if ss.ObjectMeta.Labels == nil { + ss.ObjectMeta.Labels = make(map[string]string) + } + ss.ObjectMeta.Labels[admissionWebhookAnnotationInjectKey] = sliceName + return ss } @@ -274,6 +281,12 @@ func MutateDaemonSet(ds *appsv1.DaemonSet, sliceName string) *appsv1.DaemonSet { labels[PodInjectLabelKey] = "app" labels[admissionWebhookAnnotationInjectKey] = sliceName + // add slice identifier labels to object + if ds.ObjectMeta.Labels == nil { + ds.ObjectMeta.Labels = make(map[string]string) + } + ds.ObjectMeta.Labels[admissionWebhookAnnotationInjectKey] = sliceName + return ds } @@ -304,10 +317,20 @@ func (wh *WebhookServer) ValidateServiceExport(svcex *v1beta1.ServiceExport, ctx return true, "", nil } +// returns mutationRequired bool, sliceName string func (wh *WebhookServer) MutationRequired(metadata metav1.ObjectMeta, ctx context.Context, kind string) (bool, string) { log := logger.FromContext(ctx) annotations := metadata.GetAnnotations() + labels := metadata.GetLabels() + if labels != nil { + val, exists := labels[kubesliceExcludeKey] + // don't mutate if kubeslice.io/exclude=true + if exists && val == "true" { + return false, "" + } + } + //early exit if metadata in nil //we allow empty annotation, but namespace should not be empty if metadata.GetNamespace() == "" { diff --git a/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml b/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml index 897026162..88c726df3 100644 --- a/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml +++ b/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml @@ -57,6 +57,7 @@ spec: enum: - none - istio + - envoy type: string ingress: properties: @@ -68,6 +69,19 @@ spec: enabled: type: boolean type: object + vpcServiceAccess: + properties: + egress: + properties: + enabled: + type: boolean + type: object + ingress: + properties: + enabled: + type: boolean + type: object + type: object type: object type: array maxClusters: diff --git a/tests/files/crd/networking.kubeslice.io_slices.yaml b/tests/files/crd/networking.kubeslice.io_slices.yaml index 6ac65aeb4..2dce5ac3e 100644 --- a/tests/files/crd/networking.kubeslice.io_slices.yaml +++ b/tests/files/crd/networking.kubeslice.io_slices.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.15.0 name: slices.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -22,14 +20,19 @@ spec: description: Slice is the Schema for the slices API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -104,6 +107,10 @@ spec: type: boolean type: object gatewayType: + enum: + - none + - istio + - envoy type: string ingress: properties: @@ -115,6 +122,19 @@ spec: enabled: type: boolean type: object + vpcServiceAccess: + properties: + egress: + properties: + enabled: + type: boolean + type: object + ingress: + properties: + enabled: + type: boolean + type: object + type: object type: object namespaceIsolationProfile: description: Namespace Isolation profile contains fields related @@ -208,9 +228,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml b/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml index c0cafd960..be8db8245 100644 --- a/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml +++ b/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml @@ -48,6 +48,7 @@ spec: enum: - none - istio + - envoy type: string ingress: properties: @@ -59,6 +60,19 @@ spec: enabled: type: boolean type: object + vpcServiceAccess: + properties: + egress: + properties: + enabled: + type: boolean + type: object + ingress: + properties: + enabled: + type: boolean + type: object + type: object type: object ipamClusterOctet: type: integer diff --git a/tests/spoke/slicegw_controller_test.go b/tests/spoke/slicegw_controller_test.go index dbd04bb69..59b32e608 100644 --- a/tests/spoke/slicegw_controller_test.go +++ b/tests/spoke/slicegw_controller_test.go @@ -21,6 +21,10 @@ package spoke_test import ( "context" "fmt" + "reflect" + "strconv" + "time" + kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/controllers" slicegatewaycontroller "github.com/kubeslice/worker-operator/controllers/slicegateway" @@ -35,10 +39,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" - "reflect" _ "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" - "time" ) var sliceGwFinalizer = []string{ diff --git a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go index 7cb391358..eadbcc937 100644 --- a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go +++ b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go @@ -38,6 +38,15 @@ const ( NONET NetworkType = "no-network" ) +// +kubebuilder:validation:Enum:=none;istio;envoy +type GatewayType string + +const ( + GATEWAY_TYPE_NONE GatewayType = "none" + GATEWAY_TYPE_ISTIO GatewayType = "istio" + GATEWAY_TYPE_ENVOY GatewayType = "envoy" +) + // SliceConfigSpec defines the desired state of SliceConfig type SliceConfigSpec struct { //+kubebuilder:default:=single-network @@ -69,12 +78,17 @@ type SliceConfigSpec struct { // ExternalGatewayConfig is the configuration for external gateways like 'istio', etc/ type ExternalGatewayConfig struct { - Ingress ExternalGatewayConfigOptions `json:"ingress,omitempty"` - Egress ExternalGatewayConfigOptions `json:"egress,omitempty"` - NsIngress ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` - //+kubebuilder:validation:Enum:=none;istio - GatewayType string `json:"gatewayType,omitempty"` - Clusters []string `json:"clusters,omitempty"` + Ingress ExternalGatewayConfigOptions `json:"ingress,omitempty"` + Egress ExternalGatewayConfigOptions `json:"egress,omitempty"` + NsIngress ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` + GatewayType GatewayType `json:"gatewayType,omitempty"` + Clusters []string `json:"clusters,omitempty"` + VPCServiceAccess ServiceAccess `json:"vpcServiceAccess,omitempty"` +} + +type ServiceAccess struct { + Ingress ExternalGatewayConfigOptions `json:"ingress,omitempty"` + Egress ExternalGatewayConfigOptions `json:"egress,omitempty"` } type ExternalGatewayConfigOptions struct { diff --git a/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go b/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go index 6d89ad2ab..940595dd4 100644 --- a/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go +++ b/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go @@ -90,11 +90,11 @@ type NamespaceIsolationProfile struct { } type ExternalGatewayConfig struct { - Ingress ExternalGatewayConfigOptions `json:"ingress,omitempty"` - Egress ExternalGatewayConfigOptions `json:"egress,omitempty"` - NsIngress ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` - //+kubebuilder:validation:Enum:=none;istio - GatewayType string `json:"gatewayType,omitempty"` + Ingress ExternalGatewayConfigOptions `json:"ingress,omitempty"` + Egress ExternalGatewayConfigOptions `json:"egress,omitempty"` + NsIngress ExternalGatewayConfigOptions `json:"nsIngress,omitempty"` + GatewayType controllerv1alpha1.GatewayType `json:"gatewayType,omitempty"` + VPCServiceAccess controllerv1alpha1.ServiceAccess `json:"vpcServiceAccess,omitempty"` } type ExternalGatewayConfigOptions struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 63e056bcb..dc94c1b50 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -115,7 +115,7 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/kubeslice/apis v0.3.0 +# github.com/kubeslice/apis v0.3.1 ## explicit; go 1.16 github.com/kubeslice/apis/pkg/controller/v1alpha1 github.com/kubeslice/apis/pkg/worker/v1alpha1