feat(AM-11741): protocol specification for slice gateway load balancer (

#190) * feat(AM-11741): define protocol for gw svc type * feat(AM-11741): reconcilation logic for gateway service protocol * feat(AM-11741): provide gw protocol during cert generation * fix(AM-11741): webhook validation, env var & wsc reconciler * fix(): gwsvctype mistatch --------- Signed-off-by: Mridul Gain <[email protected]>
kubeslice · Nov 3, 2023 · bd3b57c · bd3b57c
1 parent b9879df
commit bd3b57c
Show file tree

Hide file tree

Showing 16 changed files with 133 additions and 60 deletions.
diff --git a/apis/controller/v1alpha1/sliceconfig_types.go b/apis/controller/v1alpha1/sliceconfig_types.go
@@ -85,6 +85,10 @@ type SliceGatewayServiceType struct {
 	//+kubebuilder:default:=NodePort
 	//+kubebuilder:validation:Enum:=NodePort;LoadBalancer
 	Type string `json:"type"`
+	// +kubebuilder:validation:Required
+	//+kubebuilder:default:=UDP
+	//+kubebuilder:validation:Enum:=TCP;UDP
+	Protocol string `json:"protocol"`
 }
 
 // QOSProfile is the QOS Profile configuration from backend

diff --git a/apis/worker/v1alpha1/workersliceconfig_types.go b/apis/worker/v1alpha1/workersliceconfig_types.go
@@ -62,6 +62,9 @@ type WorkerSliceGatewayProvider struct {
 	//+kubebuilder:default:=NodePort
 	//+kubebuilder:validation:Enum:=NodePort;LoadBalancer
 	SliceGatewayServiceType string `json:"sliceGatewayServiceType,omitempty"`
+	//+kubebuilder:default:=UDP
+	//+kubebuilder:validation:Enum:=TCP;UDP
+	SliceGatewayProtocol string `json:"sliceGatewayProtocol,omitempty"`
 }
 
 // QOSProfile is the QOS Profile configuration from backend

diff --git a/apis/worker/v1alpha1/workerslicegateway_types.go b/apis/worker/v1alpha1/workerslicegateway_types.go
@@ -32,11 +32,14 @@ type WorkerSliceGatewaySpec struct {
 	GatewayHostType string `json:"gatewayHostType,omitempty"`
 	//+kubebuilder:default:=NodePort
 	//+kubebuilder:validation:Enum:=NodePort;LoadBalancer
-	GatewayConnectivityType string             `json:"gatewayConnectivityType,omitempty"`
-	GatewayCredentials      GatewayCredentials `json:"gatewayCredentials,omitempty"`
-	LocalGatewayConfig      SliceGatewayConfig `json:"localGatewayConfig,omitempty"`
-	RemoteGatewayConfig     SliceGatewayConfig `json:"remoteGatewayConfig,omitempty"`
-	GatewayNumber           int                `json:"gatewayNumber,omitempty"`
+	GatewayConnectivityType string `json:"gatewayConnectivityType,omitempty"`
+	//+kubebuilder:default:=UDP
+	//+kubebuilder:validation:Enum:=TCP;UDP
+	GatewayProtocol     string             `json:"gatewayProtocol,omitempty"`
+	GatewayCredentials  GatewayCredentials `json:"gatewayCredentials,omitempty"`
+	LocalGatewayConfig  SliceGatewayConfig `json:"localGatewayConfig,omitempty"`
+	RemoteGatewayConfig SliceGatewayConfig `json:"remoteGatewayConfig,omitempty"`
+	GatewayNumber       int                `json:"gatewayNumber,omitempty"`
 }
 
 type SliceGatewayConfig struct {

diff --git a/config/crd/bases/controller.kubeslice.io_sliceconfigs.yaml b/config/crd/bases/controller.kubeslice.io_sliceconfigs.yaml
@@ -164,6 +164,12 @@ spec:
                       properties:
                         cluster:
                           type: string
+                        protocol:
+                          default: UDP
+                          enum:
+                          - TCP
+                          - UDP
+                          type: string
                         type:
                           default: NodePort
                           enum:
@@ -172,6 +178,7 @@ spec:
                           type: string
                       required:
                       - cluster
+                      - protocol
                       - type
                       type: object
                     type: array

diff --git a/config/crd/bases/worker.kubeslice.io_workersliceconfigs.yaml b/config/crd/bases/worker.kubeslice.io_workersliceconfigs.yaml
@@ -117,6 +117,12 @@ spec:
                   sliceCaType:
                     default: Local
                     type: string
+                  sliceGatewayProtocol:
+                    default: UDP
+                    enum:
+                    - TCP
+                    - UDP
+                    type: string
                   sliceGatewayServiceType:
                     default: NodePort
                     enum:

diff --git a/config/crd/bases/worker.kubeslice.io_workerslicegateways.yaml b/config/crd/bases/worker.kubeslice.io_workerslicegateways.yaml
@@ -53,6 +53,12 @@ spec:
                 type: string
               gatewayNumber:
                 type: integer
+              gatewayProtocol:
+                default: UDP
+                enum:
+                - TCP
+                - UDP
+                type: string
               gatewayType:
                 default: OpenVPN
                 type: string

diff --git a/service/kube_slice_resource_names.go b/service/kube_slice_resource_names.go
@@ -151,10 +151,11 @@ var (
 )
 
 const (
-	serverGateway                  = "Server"
-	clientGateway                  = "Client"
-	workerSliceGatewayType         = "OpenVPN"
-	defaultSliceGatewayServiceType = "NodePort"
+	serverGateway                      = "Server"
+	clientGateway                      = "Client"
+	workerSliceGatewayType             = "OpenVPN"
+	defaultSliceGatewayServiceType     = "NodePort"
+	defaultSliceGatewayServiceProtocol = "UDP"
 )
 
 var (

diff --git a/service/mocks/IWorkerSliceGatewayService.go b/service/mocks/IWorkerSliceGatewayService.go
diff --git a/service/service_helper.go b/service/service_helper.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"go.uber.org/zap"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -65,3 +66,19 @@ func RemoveWorkerFinalizers(ctx context.Context, object client.Object, workerFin
 	}
 	return result
 }
+
+// get Slice gateway service type for each cluster registered with given slice
+func getSliceGwSvcTypes(sliceConfig *v1alpha1.SliceConfig) map[string]*v1alpha1.SliceGatewayServiceType {
+	var sliceGwSvcTypeMap = make(map[string]*v1alpha1.SliceGatewayServiceType)
+	for i := range sliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType {
+		gwSvctype := &sliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType[i]
+		if gwSvctype.Cluster == "*" {
+			for _, cluster := range sliceConfig.Spec.Clusters {
+				sliceGwSvcTypeMap[cluster] = gwSvctype
+			}
+		} else {
+			sliceGwSvcTypeMap[gwSvctype.Cluster] = gwSvctype
+		}
+	}
+	return sliceGwSvcTypeMap
+}
diff --git a/service/slice_config_service.go b/service/slice_config_service.go
@@ -133,17 +133,9 @@ func (s *SliceConfigService) ReconcileSliceConfig(ctx context.Context, req ctrl.
 	clusterCidr := util.FindCIDRByMaxClusters(sliceConfig.Spec.MaxClusters)
 	completeResourceName := fmt.Sprintf(util.LabelValue, util.GetObjectKind(sliceConfig), sliceConfig.GetName())
 	ownershipLabel := util.GetOwnerLabel(completeResourceName)
-	// collect cluster wise slice gw svc info
-	var sliceGwSvcTypeMap = make(map[string]*v1alpha1.SliceGatewayServiceType)
-	for _, gwSvctype := range sliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType {
-		if gwSvctype.Cluster == "*" {
-			for _, cluster := range sliceConfig.Spec.Clusters {
-				sliceGwSvcTypeMap[cluster] = &gwSvctype
-			}
-		} else {
-			sliceGwSvcTypeMap[gwSvctype.Cluster] = &gwSvctype
-		}
-	}
+	// collect slice gw svc info for given clusters
+	sliceGwSvcTypeMap := getSliceGwSvcTypes(sliceConfig)
+
 	clusterMap, err := s.ms.CreateMinimalWorkerSliceConfig(ctx, sliceConfig.Spec.Clusters, req.Namespace, ownershipLabel, sliceConfig.Name, sliceConfig.Spec.SliceSubnet, clusterCidr, sliceGwSvcTypeMap)
 	if err != nil {
 		return ctrl.Result{}, err

diff --git a/service/slice_config_webhook_validation.go b/service/slice_config_webhook_validation.go
@@ -391,7 +391,7 @@ func preventUpdate(ctx context.Context, sc *controllerv1alpha1.SliceConfig, old
 	// check new config
 	for _, new := range sc.Spec.SliceGatewayProvider.SliceGatewayServiceType {
 		oldType, exists := gwSvcType[new.Cluster]
-		if exists && new.Type != oldType {
+		if exists && oldType != defaultSliceGatewayServiceType && new.Type != oldType {
 			return field.Forbidden(field.NewPath("Spec").Child("SliceGatewayProvider").Child("SliceGatewayServiceType"), "update not allowed")
 		}
 		if !exists && new.Type != defaultSliceGatewayServiceType {

diff --git a/service/slice_config_webhook_validation_test.go b/service/slice_config_webhook_validation_test.go
@@ -142,12 +142,23 @@ func UpdateValidateSliceConfig_SliceGatewayServiceType(t *testing.T) {
 			Type:    "NodePort",
 		},
 	}
+	// loadbalancer to nodeport not allowed
 	err := ValidateSliceConfigUpdate(ctx, newSliceConfig, runtime.Object(&oldSliceConfig))
 	require.NotNil(t, err)
 	require.Contains(t, err.Error(), "Spec.SliceGatewayProvider.SliceGatewayServiceType: Forbidden:")
 	require.Contains(t, err.Error(), "update not allowed")
+
+	// 	err := ValidateSliceConfigUpdate(ctx, newSliceConfig, runtime.Object(&oldSliceConfig))
+	oldSliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType[0].Type = "NodePort"
+	newSliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType[0].Type = "LoadBalancer"
+	err = ValidateSliceConfigUpdate(ctx, newSliceConfig, runtime.Object(&oldSliceConfig))
+	require.NotNil(t, err)
+	require.NotContains(t, err.Error(), "Spec.SliceGatewayProvider.SliceGatewayServiceType: Forbidden:")
+	require.NotContains(t, err.Error(), "update not allowed")
+
 	clientMock.AssertExpectations(t)
 }
+
 func CreateValidateProjectNamespaceDoesNotExist(t *testing.T) {
 	name := "slice_config"
 	namespace := "namespace"

diff --git a/service/vpn_key_rotation_service.go b/service/vpn_key_rotation_service.go
@@ -356,7 +356,7 @@ func (v *VpnKeyRotationService) triggerJobsForCertCreation(ctx context.Context,
 			// contruct gw address
 			gatewayAddresses := v.wsgs.BuildNetworkAddresses(s.Spec.SliceSubnet, gateway.Spec.LocalGatewayConfig.ClusterName, gateway.Spec.RemoteGatewayConfig.ClusterName, clusterMap, clusterCidr)
 			// call GenerateCerts()
-			if err := v.wsgs.GenerateCerts(ctx, s.Name, s.Namespace, &gateway, cl, gatewayAddresses); err != nil {
+			if err := v.wsgs.GenerateCerts(ctx, s.Name, s.Namespace, gateway.Spec.GatewayProtocol, &gateway, cl, gatewayAddresses); err != nil {
 				return err
 			}
 		}

diff --git a/service/vpn_key_rotation_service_test.go b/service/vpn_key_rotation_service_test.go
@@ -589,7 +589,7 @@ func runReconcileVpnKeyRotationConfig(t *testing.T, tc *reconcileVpnKeyRotationC
 
 	wg.On("BuildNetworkAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gwAddress).Once()
 
-	wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+	wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 
 	clientMock.
 		On("Update", tc.updateArg1, tc.updateArg2).Return(tc.updateRet1).Once()
@@ -1103,7 +1103,7 @@ func runTriggerJobsForCertCreation(t *testing.T, tc triggerJobsForCertCreationTe
 
 	wg.On("BuildNetworkAddresses", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gwAddress).Once()
 
-	wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.generateCertsRet1).Once()
+	wg.On("GenerateCerts", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tc.generateCertsRet1).Once()
 
 	gotResp := vpn.triggerJobsForCertCreation(ctx, tc.arg1, tc.arg2)
 	require.Equal(t, gotResp, tc.expectedResp)

diff --git a/service/worker_slice_config_service.go b/service/worker_slice_config_service.go
@@ -212,21 +212,18 @@ outer:
 		logger.With(zap.Error(err)).Errorf("Failed to deep copy external gateway configuration")
 	}
 
-	// Reconcile Slice gateway service type
-	sliceGatewayProvider := workerv1alpha1.WorkerSliceGatewayProvider{
-		SliceGatewayType: sliceConfig.Spec.SliceGatewayProvider.SliceGatewayType,
-		SliceCaType:      sliceConfig.Spec.SliceGatewayProvider.SliceCaType,
-	}
-	gwSvcTypePresent := false
-	for _, gwSvcType := range sliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType {
-		if gwSvcType.Cluster == "*" || gwSvcType.Cluster == workerSliceConfig.Labels["worker-cluster"] {
-			sliceGatewayProvider.SliceGatewayServiceType = gwSvcType.Type
-			gwSvcTypePresent = true
-		}
-	}
-	if !gwSvcTypePresent {
-		sliceGatewayProvider.SliceGatewayServiceType = defaultSliceGatewayServiceType
+	// determine Slice gateway service type
+	sliceGwSvcTypeMap := getSliceGwSvcTypes(sliceConfig)
+	sliceGwSvcType := defaultSliceGatewayServiceType
+	sliceGwSvcProtocol := defaultSliceGatewayServiceProtocol
+	cluster := workerSliceConfig.Labels["worker-cluster"]
+	if val, exists := sliceGwSvcTypeMap[cluster]; exists {
+		sliceGwSvcType = val.Type
+		sliceGwSvcProtocol = val.Protocol
 	}
+	logger.Debugf("wsc %s reconciler, sliceGwSvcType %s", workerSliceConfig.Name, sliceGwSvcType)
+	logger.Debugf("slicegwsvctype map in wsc %s", sliceGwSvcTypeMap)
+	logger.Debugf("wsc reconciler cluster %s, sliceGwProtocol %s", workerSliceConfig.Labels["worker-cluster"], sliceGwSvcProtocol)
 
 	// Reconcile the Namespace Isolation Profile
 	controllerIsolationProfile := sliceConfig.Spec.NamespaceIsolationProfile
@@ -254,7 +251,8 @@ outer:
 	}
 
 	workerSliceConfig.Spec.ExternalGatewayConfig = externalGatewayConfig
-	workerSliceConfig.Spec.SliceGatewayProvider = sliceGatewayProvider
+	workerSliceConfig.Spec.SliceGatewayProvider.SliceGatewayServiceType = sliceGwSvcType
+	workerSliceConfig.Spec.SliceGatewayProvider.SliceGatewayProtocol = sliceGwSvcProtocol
 	workerSliceConfig.Spec.NamespaceIsolationProfile = workerIsolationProfile
 	workerSliceConfig.Spec.SliceName = sliceConfig.Name
 	workerSliceConfig.Spec.Octet = octet
@@ -307,9 +305,14 @@ func (s *WorkerSliceConfigService) CreateMinimalWorkerSliceConfig(ctx context.Co
 		clusterSubnetCIDR := fmt.Sprintf(util.GetClusterPrefixPool(sliceSubnet, ipamOctet, clusterCidr))
 		// determine gw svc type
 		sliceGwSvcType := defaultSliceGatewayServiceType
+		sliceGwSvcProtocol := defaultSliceGatewayServiceProtocol
 		if val, exists := sliceGwSvcTypeMap[cluster]; exists {
 			sliceGwSvcType = val.Type
+			sliceGwSvcProtocol = val.Protocol
 		}
+		logger.Debugf("setting sliceGwSvcType in create_minwsc %s", sliceGwSvcType)
+		logger.Debugf("setting sliceGwProtocol in create_minwsc %s", sliceGwSvcProtocol)
+
 		if !found {
 			label["project-namespace"] = namespace
 			label["original-slice-name"] = name
@@ -329,6 +332,7 @@ func (s *WorkerSliceConfigService) CreateMinimalWorkerSliceConfig(ctx context.Co
 			expectedSlice.Spec.ClusterSubnetCIDR = clusterSubnetCIDR
 			expectedSlice.Spec.SliceSubnet = sliceSubnet
 			expectedSlice.Spec.SliceGatewayProvider.SliceGatewayServiceType = sliceGwSvcType
+			expectedSlice.Spec.SliceGatewayProvider.SliceGatewayProtocol = sliceGwSvcProtocol
 			err = util.CreateResource(ctx, &expectedSlice)
 			if err != nil {
 				//Register an event for worker slice config creation failure
@@ -362,6 +366,7 @@ func (s *WorkerSliceConfigService) CreateMinimalWorkerSliceConfig(ctx context.Co
 			existingSlice.Spec.Octet = &ipamOctet
 			existingSlice.Spec.ClusterSubnetCIDR = clusterSubnetCIDR
 			existingSlice.Spec.SliceGatewayProvider.SliceGatewayServiceType = sliceGwSvcType
+			existingSlice.Spec.SliceGatewayProvider.SliceGatewayProtocol = sliceGwSvcProtocol
 			logger.Debug("updating slice with new octet", existingSlice)
 			if existingSlice.Annotations == nil {
 				existingSlice.Annotations = make(map[string]string)