Merge pull request #497 from matskiv/init-with-cidr

feat: init container to discover service CIDR
loft-sh · Jun 2, 2022 · a00b56f · a00b56f
2 parents d9543e3 + eb32e0f
commit a00b56f
Show file tree

Hide file tree

Showing 23 changed files with 331 additions and 136 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -18,6 +18,7 @@ COPY vendor/ vendor/
 
 # Copy the go source
 COPY cmd/vcluster cmd/vcluster
+COPY cmd/vclusterctl cmd/vclusterctl
 COPY pkg/ pkg/
 
 # Symlink /manifests folder to the synced location for development purposes

diff --git a/charts/eks/templates/api-deployment.yaml b/charts/eks/templates/api-deployment.yaml
@@ -98,7 +98,11 @@ spec:
           - '--service-account-issuer=https://kubernetes.default.svc.cluster.local'
           - '--service-account-key-file=/run/config/pki/sa.pub'
           - '--service-account-signing-key-file=/run/config/pki/sa.key'
+          {{- if .Values.serviceCIDR }}
           - '--service-cluster-ip-range={{ .Values.serviceCIDR }}'
+          {{- else }}
+          - '--service-cluster-ip-range=$(SERVICE_CIDR)'
+          {{- end }}
           - '--tls-cert-file=/run/config/pki/apiserver.crt'
           - '--tls-private-key-file=/run/config/pki/apiserver.key'
           - '--watch-cache=false'
@@ -128,7 +132,16 @@ spec:
         securityContext:
 {{ toYaml .Values.api.securityContext | indent 10 }}
         env:
+          {{- if .Values.api.env }}
 {{ toYaml .Values.api.env | indent 10 }}
+          {{- end }}
+          {{- if not .Values.serviceCIDR }}
+          - name: SERVICE_CIDR
+            valueFrom:
+              configMapKeyRef:
+                name: "vc-cidr-{{ .Release.Name }}"
+                key: cidr
+          {{- end }}
         volumeMounts:
           - mountPath: /run/config/pki
             name: certs

diff --git a/charts/eks/templates/controller-deployment.yaml b/charts/eks/templates/controller-deployment.yaml
@@ -80,7 +80,11 @@ spec:
           - '--kubeconfig=/run/config/pki/controller-manager.conf'
           - '--port=0'
           - '--profiling=false'
+          {{- if .Values.serviceCIDR }}
           - '--service-cluster-ip-range={{ .Values.serviceCIDR }}'
+          {{- else }}
+          - '--service-cluster-ip-range=$(SERVICE_CIDR)'
+          {{- end }}
           - '--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256'
           - '--use-service-account-credentials=true'
           {{- if .Values.enableHA }}
@@ -122,7 +126,16 @@ spec:
         securityContext:
 {{ toYaml .Values.controller.securityContext | indent 10 }}
         env:
+          {{- if .Values.controller.env }}
 {{ toYaml .Values.controller.env | indent 10 }}
+          {{- end }}
+          {{- if not .Values.serviceCIDR }}
+          - name: SERVICE_CIDR
+            valueFrom:
+              configMapKeyRef:
+                name: "vc-cidr-{{ .Release.Name }}"
+                key: cidr
+          {{- end }}
         volumeMounts:
           - mountPath: /run/config/pki
             name: certs

diff --git a/charts/eks/templates/pre-install-hook-job.yaml b/charts/eks/templates/pre-install-hook-job.yaml
@@ -45,5 +45,5 @@ spec:
             - --service-cidr={{ .Values.serviceCIDR }}
             {{- end }}
           resources:
-{{ toYaml .Values.etcd.resources | indent 12 }}
+{{ toYaml .Values.preInstall.resources | indent 12 }}
 {{- end }}
diff --git a/charts/eks/templates/pre-install-hook-role.yaml b/charts/eks/templates/pre-install-hook-role.yaml
@@ -10,6 +10,6 @@ metadata:
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
 rules:
   - apiGroups: [""]
-    resources: ["secrets"]
+    resources: ["secrets", "configmaps"]
     verbs: ["create", "get", "list"]
 {{- end }}
diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml
@@ -1,10 +1,3 @@
-# Make sure the service-cidr is the exact service cidr of the host cluster.
-# If this does not match, you won't be able to create services within the vcluster. You can find out
-# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster.
-# This will yield an error message in the form of: 
-# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12
-serviceCIDR: "10.96.0.0/12"
-
 # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed
 # images within the vcluster will not be rewritten.
 defaultImageRegistry: ""

diff --git a/charts/k0s/templates/secret.yaml b/charts/k0s/templates/secret.yaml
@@ -10,6 +10,9 @@ metadata:
     heritage: "{{ .Release.Service }}"
 type: Opaque
 stringData:
+  {{- if .Values.serviceCIDR }}
+  CONFIG_READY: "true"
+  {{- end }}
   {{- if .Values.config }}
   config.yaml: {{ toJson .Values.config }}
   {{- else }}
@@ -26,8 +29,12 @@ stringData:
           enable-admission-plugins: NodeRestriction
           endpoint-reconciler-type: none
       network:
-        # Will be replaced automatically from the vcluster cli
+        {{- if .Values.serviceCIDR }}
         serviceCIDR: {{ .Values.serviceCIDR }}
+        {{- else }}
+        # Will be replaced automatically by the syncer container on first 
+        serviceCIDR: CIDR_PLACEHOLDER
+        {{- end }}
         provider: custom
       controllerManager:
         extraArgs:

diff --git a/charts/k0s/templates/statefulset.yaml b/charts/k0s/templates/statefulset.yaml
@@ -96,7 +96,14 @@ spec:
           - {{ $f | quote }}
           {{- end }}
         env:
+          {{- if .Values.vcluster.env }}
 {{ toYaml .Values.vcluster.env | indent 10 }}
+          {{- end }}
+          - name: CONFIG_READY
+            valueFrom:
+              secretKeyRef:
+                name: "vc-{{ .Release.Name }}-config"
+                key: CONFIG_READY
         securityContext:
 {{ toYaml .Values.securityContext | indent 10 }}
         volumeMounts:

diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml
@@ -1,10 +1,3 @@
-# Make sure the service-cidr is the exact service cidr of the host cluster.
-# If this does not match, you won't be able to create services within the vcluster. You can find out
-# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster.
-# This will yield an error message in the form of: 
-# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12
-serviceCIDR: "10.96.0.0/12"
-
 # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed
 # images within the vcluster will not be rewritten.
 defaultImageRegistry: ""

diff --git a/charts/k3s/templates/statefulset.yaml b/charts/k3s/templates/statefulset.yaml
@@ -101,13 +101,24 @@ spec:
           {{- end }}
           {{- if .Values.serviceCIDR }}
             --service-cidr={{ .Values.serviceCIDR }}
+          {{- else }}
+            --service-cidr=$(SERVICE_CIDR)
           {{- end }}
           {{- range $f := .Values.vcluster.extraArgs }}
             {{ $f }}
           {{- end }}
             && true
         env:
+          {{- if .Values.vcluster.env }}
 {{ toYaml .Values.vcluster.env | indent 10 }}
+          {{- end }}
+          {{- if not .Values.serviceCIDR }}
+          - name: SERVICE_CIDR
+            valueFrom:
+              configMapKeyRef:
+                name: "vc-cidr-{{ .Release.Name }}"
+                key: cidr
+          {{- end }}
         securityContext:
 {{ toYaml .Values.securityContext | indent 10 }}
         volumeMounts:

diff --git a/charts/k3s/values.yaml b/charts/k3s/values.yaml
@@ -1,10 +1,3 @@
-# Make sure the service-cidr is the exact service cidr of the host cluster.
-# If this does not match, you won't be able to create services within the vcluster. You can find out
-# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster.
-# This will yield an error message in the form of: 
-# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12
-#serviceCIDR: "10.96.0.0/12"
-
 # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed
 # images within the vcluster will not be rewritten.
 defaultImageRegistry: ""
@@ -215,7 +208,7 @@ service:
   # Configuration for LoadBalancer service type
   externalIPs: []
   externalTrafficPolicy: ""
-
+  
 # Configure the ingress resource that allows you to access the vcluster
 ingress:
   # Enable ingress record generation

diff --git a/charts/k8s/templates/api-deployment.yaml b/charts/k8s/templates/api-deployment.yaml
@@ -97,7 +97,11 @@ spec:
           - '--service-account-issuer=https://kubernetes.default.svc.cluster.local'
           - '--service-account-key-file=/run/config/pki/sa.pub'
           - '--service-account-signing-key-file=/run/config/pki/sa.key'
+          {{- if .Values.serviceCIDR }}
           - '--service-cluster-ip-range={{ .Values.serviceCIDR }}'
+          {{- else }}
+          - '--service-cluster-ip-range=$(SERVICE_CIDR)'
+          {{- end }}
           - '--tls-cert-file=/run/config/pki/apiserver.crt'
           - '--tls-private-key-file=/run/config/pki/apiserver.key'
           - '--watch-cache=false'
@@ -127,7 +131,16 @@ spec:
         securityContext:
 {{ toYaml .Values.api.securityContext | indent 10 }}
         env:
+          {{- if .Values.api.env }}
 {{ toYaml .Values.api.env | indent 10 }}
+          {{- end }}
+          {{- if not .Values.serviceCIDR }}
+          - name: SERVICE_CIDR
+            valueFrom:
+              configMapKeyRef:
+                name: "vc-cidr-{{ .Release.Name }}"
+                key: cidr
+          {{- end }}
         volumeMounts:
           - mountPath: /run/config/pki
             name: certs

diff --git a/charts/k8s/templates/controller-deployment.yaml b/charts/k8s/templates/controller-deployment.yaml
@@ -89,7 +89,11 @@ spec:
           {{- end }}
           - '--horizontal-pod-autoscaler-sync-period=60s'
           - '--kubeconfig=/run/config/pki/controller-manager.conf'
+          {{- if .Values.serviceCIDR }}
           - '--service-cluster-ip-range={{ .Values.serviceCIDR }}'
+          {{- else }}
+          - '--service-cluster-ip-range=$(SERVICE_CIDR)'
+          {{- end }}
           {{- if .Values.enableHA }}
           - '--leader-elect=true'
           {{- else }}
@@ -128,7 +132,16 @@ spec:
         securityContext:
 {{ toYaml .Values.controller.securityContext | indent 10 }}
         env:
+          {{- if .Values.controller.env }}
 {{ toYaml .Values.controller.env | indent 10 }}
+          {{- end }}
+          {{- if not .Values.serviceCIDR }}
+          - name: SERVICE_CIDR
+            valueFrom:
+              configMapKeyRef:
+                name: "vc-cidr-{{ .Release.Name }}"
+                key: cidr
+          {{- end }}
         volumeMounts:
           - mountPath: /run/config/pki
             name: certs

diff --git a/charts/k8s/templates/pre-install-hook-job.yaml b/charts/k8s/templates/pre-install-hook-job.yaml
@@ -45,5 +45,5 @@ spec:
             - --service-cidr={{ .Values.serviceCIDR }}
             {{- end }}
           resources:
-{{ toYaml .Values.etcd.resources | indent 12 }}
+{{ toYaml .Values.preInstall.resources | indent 12 }}
 {{- end }}
diff --git a/charts/k8s/templates/pre-install-hook-role.yaml b/charts/k8s/templates/pre-install-hook-role.yaml
@@ -10,6 +10,6 @@ metadata:
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
 rules:
   - apiGroups: [""]
-    resources: ["secrets"]
+    resources: ["secrets", "configmaps"]
     verbs: ["create", "get", "list"]
 {{- end }}
diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml
@@ -1,10 +1,3 @@
-# Make sure the service-cidr is the exact service cidr of the host cluster.
-# If this does not match, you won't be able to create services within the vcluster. You can find out
-# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster.
-# This will yield an error message in the form of: 
-# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12
-serviceCIDR: "10.96.0.0/12"
-
 # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed
 # images within the vcluster will not be rewritten.
 defaultImageRegistry: ""

diff --git a/cmd/vcluster/cmd/certs.go b/cmd/vcluster/cmd/certs.go
@@ -2,19 +2,21 @@ package cmd
 
 import (
 	"context"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+
 	"github.com/loft-sh/vcluster/pkg/certs"
 	"github.com/loft-sh/vcluster/pkg/util/clienthelper"
+	"github.com/loft-sh/vcluster/pkg/util/servicecidr"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"io/ioutil"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
-	"path/filepath"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"strconv"
-	"strings"
 )
 
 // CertsCmd holds the certs flags
@@ -42,7 +44,7 @@ func NewCertsCommand() *cobra.Command {
 
 	cmd.Flags().StringVar(&options.ClusterName, "cluster-name", "kubernetes", "The cluster name")
 	cmd.Flags().StringVar(&options.ClusterDomain, "cluster-domain", "cluster.local", "The cluster domain ending that should be used for the virtual cluster")
-	cmd.Flags().StringVar(&options.ServiceCIDR, "service-cidr", "10.96.0.0/12", "Service CIDR is the subnet used by k8s services")
+	cmd.Flags().StringVar(&options.ServiceCIDR, "service-cidr", "", "Service CIDR is the subnet used by k8s services")
 	cmd.Flags().StringVar(&options.Prefix, "prefix", "vcluster", "Release name and prefix for generating the assets")
 	cmd.Flags().StringVar(&options.Namespace, "namespace", "", "Namespace where to deploy the cert secret to")
 	cmd.Flags().StringVar(&options.CertificateDir, "certificate-dir", "certs", "The temporary directory where the certificates will be stored")
@@ -105,6 +107,15 @@ func ExecuteCerts(options *CertsCmd) error {
 		}
 	}
 
+	cidr := options.ServiceCIDR
+	if cidr == "" {
+		cidr, err = servicecidr.EnsureServiceCIDRConfigmap(context.Background(), kubeClient, options.Namespace, options.Prefix)
+		if err != nil {
+			klog.Errorf("Failed to retrieve service CIDR range")
+			return err
+		}
+	}
+
 	secretName := options.Prefix + "-certs"
 	_, err = kubeClient.CoreV1().Secrets(options.Namespace).Get(context.Background(), secretName, metav1.GetOptions{})
 	if err == nil {
@@ -131,7 +142,7 @@ func ExecuteCerts(options *CertsCmd) error {
 		ServerCertSANs: serverSans,
 		PeerCertSANs:   serverSans,
 	}
-	cfg.Networking.ServiceSubnet = options.ServiceCIDR
+	cfg.Networking.ServiceSubnet = cidr
 	cfg.Networking.DNSDomain = options.ClusterDomain
 	cfg.ControlPlaneEndpoint = options.Prefix + "-api"
 	cfg.CertificatesDir = options.CertificateDir