Merge branch 'master' into add-UTs

Signed-off-by: Mridul Gain <[email protected]>

.github/ISSUE_TEMPLATE/FEATURE.yaml

@@ -2,7 +2,7 @@ name: 🚀 Feature
 description: "Submit a proposal for a new feature"
 title: "Feature: "
 labels: [feature]
-assignees: ["narmidm","bharath-avesha","YachikaRalhan","Rahul-D78","rahulsawra98","gourishkb","mridulgain","richiesebastian"]
+assignees: []
 body:
   - type: markdown
     attributes:

.github/workflows/scripts/start-kind.sh

@@ -8,7 +8,7 @@ ln -s /opt/kubectx/kubens /usr/local/bin/kubens
 
 # Create controller kind cluster if not present
 if [ ! $(kind get clusters | grep controller) ];then
-  kind create cluster --name controller --config .github/workflows/scripts/cluster.yaml --image kindest/node:v1.24.15
+  kind create cluster --name controller --config .github/workflows/scripts/cluster.yaml --image kindest/node:v1.29.2
 
   # Install Calico calico on controller-cluster
   echo "Installing calico on controller-cluster"
@@ -70,7 +70,7 @@ fi
 
 # Create worker1 kind cluster if not present
 if [ ! $(kind get clusters | grep worker) ];then
-  kind create cluster --name worker --config .github/workflows/scripts/cluster.yaml --image kindest/node:v1.24.15
+  kind create cluster --name worker --config .github/workflows/scripts/cluster.yaml --image kindest/node:v1.29.2
 
   # Install Calico calico on worker-cluster
   echo "Installing calico on worker-cluster"

controllers/slice/namespaces.go

@@ -89,7 +89,7 @@ func (r *SliceReconciler) reconcileAppNamespaces(ctx context.Context, slice *kub
 		log.Error(err, "Failed to list namespaces")
 		return ctrl.Result{}, err, true
 	}
-	log.Info("reconciling", "existingAppNsList", existingAppNsList)
+	debugLog.Info("reconciling", "existingAppNsList", existingAppNsList)
 	// Convert the list into a map for faster lookups. Will come in handy when we compare
 	// existing namespaces against configured namespaces.
 	existingAppNsMap := make(map[string]*nsMarker)
@@ -167,7 +167,7 @@ func (r *SliceReconciler) reconcileAllowedNamespaces(ctx context.Context, slice
 			cfgAllowedNsList = append(cfgAllowedNsList, v)
 		}
 	}
-	log.Info("reconciling", "allowedNamespaces", cfgAllowedNsList)
+	debugLog.Info("reconciling", "allowedNamespaces", cfgAllowedNsList)
 
 	// Get the list of existing namespaces that are tagged with the kube-slice label for allowed NS
 	labeledNsList := &corev1.NamespaceList{}

controllers/slicegateway/slicegateway.go

@@ -26,7 +26,6 @@ import (
 	"net"
 	"os"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -55,6 +54,7 @@ import (
 )
 
 var (
+	vpnClientFileName        = "openvpn_client.ovpn"
 	gwSidecarImage           = os.Getenv("AVESHA_GW_SIDECAR_IMAGE")
 	gwSidecarImagePullPolicy = os.Getenv("AVESHA_GW_SIDECAR_IMAGE_PULLPOLICY")
 
@@ -383,7 +383,6 @@ func (r *SliceGwReconciler) deploymentForGatewayClient(g *kubeslicev1beta1.Slice
 
 	var vpnSecretDefaultMode int32 = 0644
 
-	certFileName := "openvpn_client.ovpn"
 	sidecarImg := "nexus.dev.aveshalabs.io/kubeslice/gw-sidecar:1.0.0"
 	sidecarPullPolicy := corev1.PullAlways
 
@@ -546,23 +545,7 @@ func (r *SliceGwReconciler) deploymentForGatewayClient(g *kubeslicev1beta1.Slice
 						Command: []string{
 							"/usr/local/bin/waitForConfigToRunCmd.sh",
 						},
-						Args: []string{
-							"/vpnclient/" + certFileName,
-							"90",
-							"openvpn",
-							"--remote",
-							g.Status.Config.SliceGatewayRemoteGatewayID,
-							"--port",
-							strconv.Itoa(remotePortNumber),
-							"--ping-restart",
-							"15",
-							"--proto",
-							strings.ToLower(g.Status.Config.SliceGatewayProtocol),
-							"--txqueuelen",
-							"5000",
-							"--config",
-							"/vpnclient/" + certFileName,
-						},
+						Args: getOVPNClientContainerArgs(remotePortNumber, g),
 						SecurityContext: &corev1.SecurityContext{
 							Privileged:               &privileged,
 							AllowPrivilegeEscalation: &privileged,
@@ -1392,13 +1375,43 @@ func (r *SliceGwReconciler) ReconcileGatewayDeployments(ctx context.Context, sli
 		for _, deployment := range deployments.Items {
 			found, nodePortInUse := getClientGwRemotePortInUse(ctx, r.Client, sliceGw, deployment.Name)
 			if found {
-				_, foundInMap := gwClientToRemotePortMap.Load(deployment.Name)
-				if !foundInMap {
+				// Check if the portInUse is valid.
+				// It is valid only if the list of remoteNodePorts in the slicegw object contains the portInUse.
+				if !checkIfNodePortIsValid(sliceGw.Status.Config.SliceGatewayRemoteNodePorts, nodePortInUse) {
+					// Get a valid port number for this deployment
+					portNumToUpdate, err := allocateNodePortToClient(sliceGw.Status.Config.SliceGatewayRemoteNodePorts, deployment.Name, &gwClientToRemotePortMap)
+					if err != nil {
+						return ctrl.Result{}, err, true
+					}
+					// Update the port map
+					gwClientToRemotePortMap.Store(deployment.Name, portNumToUpdate)
+					err = r.updateGatewayDeploymentNodePort(ctx, r.Client, sliceGw, &deployment, portNumToUpdate)
+					if err != nil {
+						return ctrl.Result{}, err, true
+					}
+					// Requeue if the update was successful
+					return ctrl.Result{}, nil, true
+				}
+				// At this point, the node port in use is valid. Check if the port map is populated. Only case
+				// where it is not populated is if the operator restarts. The populated value must match the
+				// port in use. If not, the deploy needs to be updated to match the state stored in the operator.
+				portInMap, foundInMap := gwClientToRemotePortMap.Load(deployment.Name)
+				if foundInMap {
+					if portInMap != nodePortInUse {
+						// Update the deployment since the port numbers do not match
+						err := r.updateGatewayDeploymentNodePort(ctx, r.Client, sliceGw, &deployment, portInMap.(int))
+						if err != nil {
+							return ctrl.Result{}, err, true
+						}
+						// Requeue if the update was successful
+						return ctrl.Result{}, nil, true
+					}
+				} else {
 					gwClientToRemotePortMap.Store(deployment.Name, nodePortInUse)
 				}
-				// TODO: Handle the case of the port number in the deployment and the one in the port map being different
 			}
 		}
+
 	}
 
 	// Delete any deployments marked for deletion. We could have an external orchestrator (like the workerslicegatewayrecycler) request
@@ -1599,3 +1612,30 @@ func (r *SliceGwReconciler) createPodDisruptionBudgetForSliceGatewayPods(ctx con
 
 	return nil
 }
+
+// updateGatewayDeploymentNodePort updates the gateway client deployments with the relevant updated ports
+// from the workersliceconfig
+func (r *SliceGwReconciler) updateGatewayDeploymentNodePort(ctx context.Context, c client.Client, g *kubeslicev1beta1.SliceGateway, deployment *appsv1.Deployment, nodePort int) error {
+	containers := deployment.Spec.Template.Spec.Containers
+	for contIndex, cont := range containers {
+		if cont.Name == "kubeslice-sidecar" {
+			for index, key := range cont.Env {
+				if key.Name == "NODE_PORT" {
+					updatedEnvVar := corev1.EnvVar{
+						Name:  "NODE_PORT",
+						Value: strconv.Itoa(nodePort),
+					}
+					cont.Env[index] = updatedEnvVar
+				}
+			}
+		} else if cont.Name == "kubeslice-openvpn-client" {
+			containers[contIndex].Args = getOVPNClientContainerArgs(nodePort, g)
+		}
+	}
+	deployment.Spec.Template.Spec.Containers = containers
+	err := r.Update(ctx, deployment)
+	if err != nil {
+		return err
+	}
+	return nil
+}

controllers/slicegateway/utils.go

@@ -22,16 +22,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
 	gwsidecarpb "github.com/kubeslice/gateway-sidecar/pkg/sidecar/sidecarpb"
 	kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1"
 	"github.com/kubeslice/worker-operator/controllers"
 	ossEvents "github.com/kubeslice/worker-operator/events"
 	"github.com/kubeslice/worker-operator/pkg/utils"
 	webhook "github.com/kubeslice/worker-operator/pkg/webhook/pod"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -77,15 +77,24 @@ func getGwSvcNameFromDepName(depName string) string {
 	return "svc-" + depName
 }
 
-func contains(s []string, e string) bool {
-	for _, a := range s {
-		if a == e {
+func contains[T comparable](s []T, e T) bool {
+	for _, element := range s {
+		if element == e {
 			return true
 		}
 	}
 	return false
 }
 
+func containsWithIndex[T comparable](s []T, e T) (bool, int) {
+	for index, element := range s {
+		if element == e {
+			return true, index
+		}
+	}
+	return false, 0
+}
+
 func getPodIPs(slicegateway *kubeslicev1beta1.SliceGateway) []string {
 	podIPs := make([]string, 0)
 	for i := range slicegateway.Status.GatewayPodStatus {
@@ -426,3 +435,44 @@ func (r *SliceGwReconciler) cleanupSliceGwResources(ctx context.Context, slicegw
 
 	return nil
 }
+
+// getOVPNClientContainerArgs returns the args needed for the ovpn client deployment container
+func getOVPNClientContainerArgs(remotePortNumber int, g *kubeslicev1beta1.SliceGateway) []string {
+	args := []string{
+		"/vpnclient/" + vpnClientFileName,
+		"90",
+		"openvpn",
+		"--remote",
+		g.Status.Config.SliceGatewayRemoteGatewayID,
+		"--port",
+		strconv.Itoa(remotePortNumber),
+		"--ping-restart",
+		"15",
+		"--proto",
+		strings.ToLower(g.Status.Config.SliceGatewayProtocol),
+		"--txqueuelen",
+		"5000",
+		"--config",
+		"/vpnclient/" + vpnClientFileName,
+	}
+	return args
+}
+
+// a helper to assign distinct port to each client deployment
+func allocateNodePortToClient(correctNodePorts []int, depName string, nodePortsMap *sync.Map) (int, error) {
+	nodePortsMap.Range(func(k, v interface{}) bool {
+		if ok, index := containsWithIndex(correctNodePorts, v.(int)); ok {
+			correctNodePorts = append(correctNodePorts[:index], correctNodePorts[index+1:]...)
+		}
+		return true
+	})
+	if len(correctNodePorts) > 0 {
+		return correctNodePorts[0], nil
+	} else {
+		port, ok := nodePortsMap.Load(depName)
+		if ok {
+			return port.(int), nil
+		}
+		return 0, errors.New("could not allocate a port")
+	}
+}

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/kubeslice/apis v0.3.0
 	github.com/kubeslice/gateway-sidecar v0.2.0
-	github.com/kubeslice/kubeslice-monitoring v0.1.10
+	github.com/kubeslice/kubeslice-monitoring v0.2.1
 	github.com/kubeslice/netops v0.1.3
 	github.com/kubeslice/router-sidecar v1.4.0
 	github.com/kubeslice/slicegw-edge v1.0.0

go.sum

@@ -385,8 +385,8 @@ github.com/kubeslice/apis v0.3.0 h1:sSaMLKWN9OfUdcTYAcIaUpRwlZqrmgKLPG5I8dYYH2k=
 github.com/kubeslice/apis v0.3.0/go.mod h1:YDSfpIsQM+FtQPaZVGNCTZnlp3viWuQhkjJjIHQdaYs=
 github.com/kubeslice/gateway-sidecar v0.2.0 h1:Ja3fIUivuSjUFQ4lPCt79ATq99BxslvAFYUwV9Urpy4=
 github.com/kubeslice/gateway-sidecar v0.2.0/go.mod h1:nM1+Wjud2vk44cUg+9iwBbWTpqI+2Ecbn9NuaHEs9aY=
-github.com/kubeslice/kubeslice-monitoring v0.1.10 h1:ozzGuSxr5dsouI2ATtPOLygsKfQ8w6wUI9QHkSp33TQ=
-github.com/kubeslice/kubeslice-monitoring v0.1.10/go.mod h1:Ux4e/ZvTKru8ewnOP0YudLey6ZItriCx6Tzjt9oVnHg=
+github.com/kubeslice/kubeslice-monitoring v0.2.1 h1:wtmIEigpQoKzuckof7QRqdsaa4lV/rqxd/FcmOj5N5Q=
+github.com/kubeslice/kubeslice-monitoring v0.2.1/go.mod h1:PASwPHqwCjfL838gC6cSiGNu9aML9RTvDqyTpbC1FWo=
 github.com/kubeslice/netops v0.1.3 h1:XT9RDxIwh/6381c/GwZi1vHCw8D0ZKQUPISok326/Aw=
 github.com/kubeslice/netops v0.1.3/go.mod h1:ZHUN74gCIkjtJO2BS2w0+JvTi+366JxxgCpU3OvLDk0=
 github.com/kubeslice/router-sidecar v1.4.0 h1:UX2HBoLfFhc2b5mdFx9pj3kQZJffav70zaiPUWcYu1s=

pkg/hub/controllers/cluster/reconciler.go

@@ -97,7 +97,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
 		return reconcile.Result{}, err
 	}
 
-	log.Info("got cluster CR from hub", "cluster", cr)
+	log.Info("got cluster CR from hub", "cluster", cr.Name)
 	requeue, result, err := r.handleClusterDeletion(cr, ctx, req)
 	if requeue {
 		return result, err

pkg/hub/controllers/slice_controller.go

@@ -245,7 +245,7 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req reconcile.Request)
 			return reconcile.Result{}, err
 		} else {
 			utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventWorkerSliceHealthUpdated, sliceControllerName)
-			log.Info("succesfully updated the slice CR ", "slice CR ", slice)
+			log.Info("succesfully updated the slice CR ", "slice CR ", slice.Name)
 		}
 	}
 	r.counterSliceUpdated.WithLabelValues(sliceName).Add(1)