Merge pull request #28 from FabianKramm/main

Chart Improvements & Leader Election

chart/templates/rbac/clusterrole.yaml

@@ -13,7 +13,7 @@ rules:
     resources: ["nodes", "nodes/status"]
     verbs: ["get", "watch", "list", "update", "patch"]
   - apiGroups: [""]
-    resources: ["pods", "nodes/proxy", "persistentvolumes"]
+    resources: ["pods", "nodes/proxy", "nodes/metrics", "nodes/stats", "persistentvolumes"]
     verbs: ["get", "watch", "list"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]

chart/templates/service.yaml

@@ -9,7 +9,7 @@ metadata:
     release: "{{ .Release.Name }}"
     heritage: "{{ .Release.Service }}"
 spec:
-  type: ClusterIP
+  type: {{ .Values.service.type }}
   ports:
     - name: https
       port: 443

chart/templates/statefulset.yaml

@@ -10,11 +10,13 @@ metadata:
     heritage: "{{ .Release.Service }}"
 spec:
   serviceName: {{ .Release.Name }}-headless
-  replicas: 1
+  replicas: {{ .Values.replicas }}
   selector:
     matchLabels:
       app: vcluster
       release: {{ .Release.Name }}
+  {{- if .Values.storage.persistence }}
+  {{- if not .Values.storage.volumeClaimTemplates }}
   volumeClaimTemplates:
     - metadata:
         name: data
@@ -24,6 +26,11 @@ spec:
         resources:
           requests:
             storage: {{ .Values.storage.size }}
+  {{- else }}
+  volumeClaimTemplates:
+{{ toYaml .Values.volumeClaimTemplates | indent 4 }}
+  {{- end }}
+  {{- end }}
   template:
     metadata:
       labels:
@@ -36,6 +43,16 @@ spec:
       {{- else }}
       serviceAccountName: vc-{{ .Release.Name }}
       {{- end }}
+      {{- if .Values.volumes }}
+      volumes:
+{{ toYaml .Values.volumes | indent 8 }}
+      {{- else }}
+      {{- if not .Values.storage.persistence }}
+      volumes:
+      - name: data
+        emptyDir: {}
+      {{- end }}
+      {{- end }}
       containers:
       - image: {{ .Values.vcluster.image }}
         name: vcluster

chart/values.yaml

@@ -1,15 +1,17 @@
-# Default values for vcluster.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
+# Syncer configuration
 syncer:
+  # Image to use for the syncer
+  # image: loftsh/vcluster
   extraArgs: []
   env: []
   volumeMounts:
     - mountPath: /data
       name: data
   resources: {}
 
+# Virtual Cluster (k3s) configuration
 vcluster:
+  # Image to use for the virtual cluster
   image: rancher/k3s:v1.21.0-k3s1
   command:
     - /bin/k3s
@@ -24,6 +26,11 @@ vcluster:
     - --disable-cloud-controller
     - --flannel-backend=none
     - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle
+  # Extra arguments for k3s. Make sure the service-cidr is the exact service cidr of the host cluster.
+  # If this does not match, you won't be able to create services within the vcluster. You can find out
+  # the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster.
+  # This will yield an error message in the form of: 
+  # The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12
   extraArgs:
     - --service-cidr=10.96.0.0/12
   volumeMounts:
@@ -32,15 +39,36 @@ vcluster:
   env: []
   resources: {}
 
+# Storage settings for the vcluster
 storage:
+  # If this is disabled, vcluster will use an emptyDir instead
+  # of a PersistentVolumeClaim
+  persistence: true
+  # Size of the persistent volume claim
   size: 5Gi
-  mountPath: /data
+
+# Extra volumes that should be created for the StatefulSet
+volumes: []
 
+# Service account that should be used by the vcluster
 serviceAccount:
   create: true
+  # Optional name of the service account to use
+  # name: default
 
+# Roles & ClusterRoles for the vcluster
 rbac:
   clusterRole:
+    # Enable this to let the vcluster sync
+    # real nodes, storage classes and priority classes
     create: false
   role:
+    # This is required for basic functionality of vcluster
     create: true
+
+# The amount of replicas to run the statefulset with
+replicas: 1
+
+# Service configurations
+service:
+  type: ClusterIP

cmd/vcluster/context/context.go

@@ -46,6 +46,10 @@ type VirtualClusterOptions struct {
 	OverrideHostsContainerImage string
 
 	ClusterDomain string
+
+	LeaseDuration int64
+	RenewDeadline int64
+	RetryPeriod   int64
 }
 
 type ControllerContext struct {

cmd/vcluster/main.go

@@ -5,7 +5,7 @@ import (
 	"github.com/loft-sh/vcluster/pkg/apis"
 	"github.com/loft-sh/vcluster/pkg/controllers/resources/nodes"
 	"github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice"
-	"github.com/loft-sh/vcluster/pkg/indices"
+	"github.com/loft-sh/vcluster/pkg/leaderelection"
 	"io/ioutil"
 	"k8s.io/client-go/discovery"
 	"os"
@@ -118,6 +118,9 @@ func NewCommand() *cobra.Command {
 	cmd.Flags().StringVar(&options.OverrideHostsContainerImage, "override-hosts-container-image", pods.HostsRewriteImage, "The image for the init container that is used for creating the override hosts file.")
 
 	cmd.Flags().StringVar(&options.ClusterDomain, "cluster-domain", "cluster.local", "The cluster domain ending that should be used for the virtual cluster")
+	cmd.Flags().Int64Var(&options.LeaseDuration, "lease-duration", 60, "Lease duration of the leader election in seconds")
+	cmd.Flags().Int64Var(&options.RenewDeadline, "renew-deadline", 40, "Renew deadline of the leader election in seconds")
+	cmd.Flags().Int64Var(&options.RetryPeriod, "retry-period", 15, "Retry period of the leader election in seconds")
 	return cmd
 }
 
@@ -255,20 +258,8 @@ func Execute(cobraCmd *cobra.Command, args []string, options *context.VirtualClu
 		return errors.Wrap(err, "create controller context")
 	}
 
-	// make sure the kubernetes service is synced
-	err = syncKubernetesService(ctx)
-	if err != nil {
-		return errors.Wrap(err, "sync kubernetes service")
-	}
-
-	// register the extra indices
-	err = indices.AddIndices(ctx)
-	if err != nil {
-		return errors.Wrap(err, "register extra indices")
-	}
-
-	// register the controllers
-	err = controllers.Register(ctx)
+	// register the indices
+	err = controllers.RegisterIndices(ctx)
 	if err != nil {
 		return errors.Wrap(err, "register controllers")
 	}
@@ -293,9 +284,37 @@ func Execute(cobraCmd *cobra.Command, args []string, options *context.VirtualClu
 	localManager.GetCache().WaitForCacheSync(ctx.Context)
 	virtualClusterManager.GetCache().WaitForCacheSync(ctx.Context)
 
-	// start the node service provider
+	// start leader election for controllers
 	go func() {
-		ctx.NodeServiceProvider.Start(ctx.Context)
+		err = leaderelection.StartLeaderElection(ctx, scheme, func() error {
+			// make sure the kubernetes service is synced
+			err = syncKubernetesService(ctx)
+			if err != nil {
+				return errors.Wrap(err, "sync kubernetes service")
+			}
+
+			// start the node service provider
+			go func() {
+				ctx.NodeServiceProvider.Start(ctx.Context)
+			}()
+
+			// register controllers
+			err := controllers.RegisterControllers(ctx)
+			if err != nil {
+				return err
+			}
+
+			// write the kube config to secret
+			err = writeKubeConfigToSecret(ctx, &rawConfig)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		})
+		if err != nil {
+			klog.Fatalf("Error starting leader election: %v", err)
+		}
 	}()
 
 	// start the proxy
@@ -304,11 +323,6 @@ func Execute(cobraCmd *cobra.Command, args []string, options *context.VirtualClu
 		return err
 	}
 
-	err = writeKubeConfigToSecret(ctx, &rawConfig)
-	if err != nil {
-		return err
-	}
-
 	// start the proxy server in secure mode
 	err = proxyServer.ServeOnListenerTLS(options.BindAddress, options.Port, ctx.StopChan)
 	if err != nil {

cmd/vclusterctl/cmd/connect.go

@@ -30,7 +30,8 @@ type ConnectCmd struct {
 	Print         bool
 	LocalPort     int
 
-	log log.Logger
+	Server string
+	log    log.Logger
 }
 
 // NewConnectCmd creates a new command
@@ -67,6 +68,7 @@ vcluster connect test --namespace test
 	cobraCmd.Flags().BoolVar(&cmd.Print, "print", false, "When enabled prints the context to stdout")
 	cobraCmd.Flags().StringVarP(&cmd.Namespace, "namespace", "n", "", "The namespace the vcluster is in")
 	cobraCmd.Flags().StringVar(&cmd.PodName, "pod", "", "The pod to connect to")
+	cobraCmd.Flags().StringVar(&cmd.Server, "server", "", "The server to connect to")
 	cobraCmd.Flags().IntVar(&cmd.LocalPort, "local-port", 8443, "The local port to forward the virtual cluster to")
 	return cobraCmd
 }
@@ -125,14 +127,18 @@ func (cmd *ConnectCmd) Run(cobraCmd *cobra.Command, args []string) error {
 
 	port := ""
 	for k := range kubeConfig.Clusters {
-		splitted := strings.Split(kubeConfig.Clusters[k].Server, ":")
-		if len(splitted) != 3 {
-			return fmt.Errorf("unexpected server in kubeconfig: %s", kubeConfig.Clusters[k].Server)
-		}
+		if cmd.Server != "" {
+			kubeConfig.Clusters[k].Server = cmd.Server
+		} else {
+			splitted := strings.Split(kubeConfig.Clusters[k].Server, ":")
+			if len(splitted) != 3 {
+				return fmt.Errorf("unexpected server in kubeconfig: %s", kubeConfig.Clusters[k].Server)
+			}
 
-		port = splitted[2]
-		splitted[2] = strconv.Itoa(cmd.LocalPort)
-		kubeConfig.Clusters[k].Server = strings.Join(splitted, ":")
+			port = splitted[2]
+			splitted[2] = strconv.Itoa(cmd.LocalPort)
+			kubeConfig.Clusters[k].Server = strings.Join(splitted, ":")
+		}
 	}
 
 	out, err = clientcmd.Write(*kubeConfig)
@@ -178,8 +184,11 @@ func (cmd *ConnectCmd) Run(cobraCmd *cobra.Command, args []string) error {
 		cmd.log.Donef("Virtual cluster kube config written to: %s. You can access the cluster via `kubectl --kubeconfig %s get namespaces`", cmd.KubeConfig, cmd.KubeConfig)
 	}
 
-	forwardPorts := strconv.Itoa(cmd.LocalPort) + ":" + port
+	if cmd.Server != "" {
+		return nil
+	}
 
+	forwardPorts := strconv.Itoa(cmd.LocalPort) + ":" + port
 	command := []string{"kubectl", "port-forward", "--namespace", cmd.Namespace, podName, forwardPorts}
 	cmd.log.Infof("Starting port forwarding: %s", strings.Join(command, " "))
 	portforwardCmd := exec.Command(command[0], command[1:]...)