diff --git a/Dockerfile b/Dockerfile index d1460a9bf3..e3b317904b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY vendor/ vendor/ # Copy the go source COPY cmd/vcluster cmd/vcluster +COPY cmd/vclusterctl cmd/vclusterctl COPY pkg/ pkg/ # Symlink /manifests folder to the synced location for development purposes diff --git a/charts/eks/templates/api-deployment.yaml b/charts/eks/templates/api-deployment.yaml index 8a5fc8c564..75d77cc78a 100644 --- a/charts/eks/templates/api-deployment.yaml +++ b/charts/eks/templates/api-deployment.yaml @@ -98,7 +98,11 @@ spec: - '--service-account-issuer=https://kubernetes.default.svc.cluster.local' - '--service-account-key-file=/run/config/pki/sa.pub' - '--service-account-signing-key-file=/run/config/pki/sa.key' + {{- if .Values.serviceCIDR }} - '--service-cluster-ip-range={{ .Values.serviceCIDR }}' + {{- else }} + - '--service-cluster-ip-range=$(SERVICE_CIDR)' + {{- end }} - '--tls-cert-file=/run/config/pki/apiserver.crt' - '--tls-private-key-file=/run/config/pki/apiserver.key' - '--watch-cache=false' @@ -128,7 +132,16 @@ spec: securityContext: {{ toYaml .Values.api.securityContext | indent 10 }} env: + {{- if .Values.api.env }} {{ toYaml .Values.api.env | indent 10 }} + {{- end }} + {{- if not .Values.serviceCIDR }} + - name: SERVICE_CIDR + valueFrom: + configMapKeyRef: + name: "vc-cidr-{{ .Release.Name }}" + key: cidr + {{- end }} volumeMounts: - mountPath: /run/config/pki name: certs diff --git a/charts/eks/templates/controller-deployment.yaml b/charts/eks/templates/controller-deployment.yaml index 0c29dfe244..c751e6ae8d 100644 --- a/charts/eks/templates/controller-deployment.yaml +++ b/charts/eks/templates/controller-deployment.yaml @@ -80,7 +80,11 @@ spec: - '--kubeconfig=/run/config/pki/controller-manager.conf' - '--port=0' - '--profiling=false' + {{- if .Values.serviceCIDR }} - '--service-cluster-ip-range={{ .Values.serviceCIDR }}' + {{- else }} + - '--service-cluster-ip-range=$(SERVICE_CIDR)' + {{- end }} - '--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' - '--use-service-account-credentials=true' {{- if .Values.enableHA }} @@ -122,7 +126,16 @@ spec: securityContext: {{ toYaml .Values.controller.securityContext | indent 10 }} env: + {{- if .Values.controller.env }} {{ toYaml .Values.controller.env | indent 10 }} + {{- end }} + {{- if not .Values.serviceCIDR }} + - name: SERVICE_CIDR + valueFrom: + configMapKeyRef: + name: "vc-cidr-{{ .Release.Name }}" + key: cidr + {{- end }} volumeMounts: - mountPath: /run/config/pki name: certs diff --git a/charts/eks/templates/pre-install-hook-job.yaml b/charts/eks/templates/pre-install-hook-job.yaml index d7c98142fd..5c0677cf91 100644 --- a/charts/eks/templates/pre-install-hook-job.yaml +++ b/charts/eks/templates/pre-install-hook-job.yaml @@ -45,5 +45,5 @@ spec: - --service-cidr={{ .Values.serviceCIDR }} {{- end }} resources: -{{ toYaml .Values.etcd.resources | indent 12 }} +{{ toYaml .Values.preInstall.resources | indent 12 }} {{- end }} \ No newline at end of file diff --git a/charts/eks/templates/pre-install-hook-role.yaml b/charts/eks/templates/pre-install-hook-role.yaml index 2bc7869bca..e304b9a2a5 100644 --- a/charts/eks/templates/pre-install-hook-role.yaml +++ b/charts/eks/templates/pre-install-hook-role.yaml @@ -10,6 +10,6 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed rules: - apiGroups: [""] - resources: ["secrets"] + resources: ["secrets", "configmaps"] verbs: ["create", "get", "list"] {{- end }} \ No newline at end of file diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml index ae6e7a49f8..b14823f57d 100644 --- a/charts/eks/values.yaml +++ b/charts/eks/values.yaml @@ -1,10 +1,3 @@ -# Make sure the service-cidr is the exact service cidr of the host cluster. -# If this does not match, you won't be able to create services within the vcluster. You can find out -# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster. -# This will yield an error message in the form of: -# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12 -serviceCIDR: "10.96.0.0/12" - # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed # images within the vcluster will not be rewritten. defaultImageRegistry: "" diff --git a/charts/k0s/templates/secret.yaml b/charts/k0s/templates/secret.yaml index a93d75c8cf..f2642a8193 100644 --- a/charts/k0s/templates/secret.yaml +++ b/charts/k0s/templates/secret.yaml @@ -10,6 +10,9 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque stringData: + {{- if .Values.serviceCIDR }} + CONFIG_READY: "true" + {{- end }} {{- if .Values.config }} config.yaml: {{ toJson .Values.config }} {{- else }} @@ -26,8 +29,12 @@ stringData: enable-admission-plugins: NodeRestriction endpoint-reconciler-type: none network: - # Will be replaced automatically from the vcluster cli + {{- if .Values.serviceCIDR }} serviceCIDR: {{ .Values.serviceCIDR }} + {{- else }} + # Will be replaced automatically by the syncer container on first + serviceCIDR: CIDR_PLACEHOLDER + {{- end }} provider: custom controllerManager: extraArgs: diff --git a/charts/k0s/templates/statefulset.yaml b/charts/k0s/templates/statefulset.yaml index 487e662eda..e55c90cd04 100644 --- a/charts/k0s/templates/statefulset.yaml +++ b/charts/k0s/templates/statefulset.yaml @@ -96,7 +96,14 @@ spec: - {{ $f | quote }} {{- end }} env: + {{- if .Values.vcluster.env }} {{ toYaml .Values.vcluster.env | indent 10 }} + {{- end }} + - name: CONFIG_READY + valueFrom: + secretKeyRef: + name: "vc-{{ .Release.Name }}-config" + key: CONFIG_READY securityContext: {{ toYaml .Values.securityContext | indent 10 }} volumeMounts: diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml index 8fae00b890..92b7e11f67 100644 --- a/charts/k0s/values.yaml +++ b/charts/k0s/values.yaml @@ -1,10 +1,3 @@ -# Make sure the service-cidr is the exact service cidr of the host cluster. -# If this does not match, you won't be able to create services within the vcluster. You can find out -# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster. -# This will yield an error message in the form of: -# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12 -serviceCIDR: "10.96.0.0/12" - # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed # images within the vcluster will not be rewritten. defaultImageRegistry: "" diff --git a/charts/k3s/templates/statefulset.yaml b/charts/k3s/templates/statefulset.yaml index f5121d6bc8..f36c05f56a 100644 --- a/charts/k3s/templates/statefulset.yaml +++ b/charts/k3s/templates/statefulset.yaml @@ -101,13 +101,24 @@ spec: {{- end }} {{- if .Values.serviceCIDR }} --service-cidr={{ .Values.serviceCIDR }} + {{- else }} + --service-cidr=$(SERVICE_CIDR) {{- end }} {{- range $f := .Values.vcluster.extraArgs }} {{ $f }} {{- end }} && true env: + {{- if .Values.vcluster.env }} {{ toYaml .Values.vcluster.env | indent 10 }} + {{- end }} + {{- if not .Values.serviceCIDR }} + - name: SERVICE_CIDR + valueFrom: + configMapKeyRef: + name: "vc-cidr-{{ .Release.Name }}" + key: cidr + {{- end }} securityContext: {{ toYaml .Values.securityContext | indent 10 }} volumeMounts: diff --git a/charts/k3s/values.yaml b/charts/k3s/values.yaml index 75d2ced40f..111c93ebd3 100644 --- a/charts/k3s/values.yaml +++ b/charts/k3s/values.yaml @@ -1,10 +1,3 @@ -# Make sure the service-cidr is the exact service cidr of the host cluster. -# If this does not match, you won't be able to create services within the vcluster. You can find out -# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster. -# This will yield an error message in the form of: -# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12 -#serviceCIDR: "10.96.0.0/12" - # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed # images within the vcluster will not be rewritten. defaultImageRegistry: "" @@ -215,7 +208,7 @@ service: # Configuration for LoadBalancer service type externalIPs: [] externalTrafficPolicy: "" - + # Configure the ingress resource that allows you to access the vcluster ingress: # Enable ingress record generation diff --git a/charts/k8s/templates/api-deployment.yaml b/charts/k8s/templates/api-deployment.yaml index 7585a9a61c..f202d14fa8 100644 --- a/charts/k8s/templates/api-deployment.yaml +++ b/charts/k8s/templates/api-deployment.yaml @@ -97,7 +97,11 @@ spec: - '--service-account-issuer=https://kubernetes.default.svc.cluster.local' - '--service-account-key-file=/run/config/pki/sa.pub' - '--service-account-signing-key-file=/run/config/pki/sa.key' + {{- if .Values.serviceCIDR }} - '--service-cluster-ip-range={{ .Values.serviceCIDR }}' + {{- else }} + - '--service-cluster-ip-range=$(SERVICE_CIDR)' + {{- end }} - '--tls-cert-file=/run/config/pki/apiserver.crt' - '--tls-private-key-file=/run/config/pki/apiserver.key' - '--watch-cache=false' @@ -127,7 +131,16 @@ spec: securityContext: {{ toYaml .Values.api.securityContext | indent 10 }} env: + {{- if .Values.api.env }} {{ toYaml .Values.api.env | indent 10 }} + {{- end }} + {{- if not .Values.serviceCIDR }} + - name: SERVICE_CIDR + valueFrom: + configMapKeyRef: + name: "vc-cidr-{{ .Release.Name }}" + key: cidr + {{- end }} volumeMounts: - mountPath: /run/config/pki name: certs diff --git a/charts/k8s/templates/controller-deployment.yaml b/charts/k8s/templates/controller-deployment.yaml index 2dd3ddce52..8624fe3f30 100644 --- a/charts/k8s/templates/controller-deployment.yaml +++ b/charts/k8s/templates/controller-deployment.yaml @@ -89,7 +89,11 @@ spec: {{- end }} - '--horizontal-pod-autoscaler-sync-period=60s' - '--kubeconfig=/run/config/pki/controller-manager.conf' + {{- if .Values.serviceCIDR }} - '--service-cluster-ip-range={{ .Values.serviceCIDR }}' + {{- else }} + - '--service-cluster-ip-range=$(SERVICE_CIDR)' + {{- end }} {{- if .Values.enableHA }} - '--leader-elect=true' {{- else }} @@ -128,7 +132,16 @@ spec: securityContext: {{ toYaml .Values.controller.securityContext | indent 10 }} env: + {{- if .Values.controller.env }} {{ toYaml .Values.controller.env | indent 10 }} + {{- end }} + {{- if not .Values.serviceCIDR }} + - name: SERVICE_CIDR + valueFrom: + configMapKeyRef: + name: "vc-cidr-{{ .Release.Name }}" + key: cidr + {{- end }} volumeMounts: - mountPath: /run/config/pki name: certs diff --git a/charts/k8s/templates/pre-install-hook-job.yaml b/charts/k8s/templates/pre-install-hook-job.yaml index d7c98142fd..5c0677cf91 100644 --- a/charts/k8s/templates/pre-install-hook-job.yaml +++ b/charts/k8s/templates/pre-install-hook-job.yaml @@ -45,5 +45,5 @@ spec: - --service-cidr={{ .Values.serviceCIDR }} {{- end }} resources: -{{ toYaml .Values.etcd.resources | indent 12 }} +{{ toYaml .Values.preInstall.resources | indent 12 }} {{- end }} \ No newline at end of file diff --git a/charts/k8s/templates/pre-install-hook-role.yaml b/charts/k8s/templates/pre-install-hook-role.yaml index 2bc7869bca..e304b9a2a5 100644 --- a/charts/k8s/templates/pre-install-hook-role.yaml +++ b/charts/k8s/templates/pre-install-hook-role.yaml @@ -10,6 +10,6 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed rules: - apiGroups: [""] - resources: ["secrets"] + resources: ["secrets", "configmaps"] verbs: ["create", "get", "list"] {{- end }} \ No newline at end of file diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml index 667dce6a28..1fa4422bde 100644 --- a/charts/k8s/values.yaml +++ b/charts/k8s/values.yaml @@ -1,10 +1,3 @@ -# Make sure the service-cidr is the exact service cidr of the host cluster. -# If this does not match, you won't be able to create services within the vcluster. You can find out -# the service cidr of the host cluster by creating a service with a not allowed ClusterIP in the host cluster. -# This will yield an error message in the form of: -# The Service "faulty-service" is invalid: spec.clusterIP: Invalid value: "1.1.1.1": provided IP is not in the valid range. The range of valid IPs is 10.96.0.0/12 -serviceCIDR: "10.96.0.0/12" - # DefaultImageRegistry will be prepended to all deployed vcluster images, such as the vcluster pod, coredns etc.. Deployed # images within the vcluster will not be rewritten. defaultImageRegistry: "" diff --git a/cmd/vcluster/cmd/certs.go b/cmd/vcluster/cmd/certs.go index c9b0da5089..4eae1727f0 100644 --- a/cmd/vcluster/cmd/certs.go +++ b/cmd/vcluster/cmd/certs.go @@ -2,19 +2,21 @@ package cmd import ( "context" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "github.com/loft-sh/vcluster/pkg/certs" "github.com/loft-sh/vcluster/pkg/util/clienthelper" + "github.com/loft-sh/vcluster/pkg/util/servicecidr" "github.com/pkg/errors" "github.com/spf13/cobra" - "io/ioutil" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog" - "path/filepath" ctrl "sigs.k8s.io/controller-runtime" - "strconv" - "strings" ) // CertsCmd holds the certs flags @@ -42,7 +44,7 @@ func NewCertsCommand() *cobra.Command { cmd.Flags().StringVar(&options.ClusterName, "cluster-name", "kubernetes", "The cluster name") cmd.Flags().StringVar(&options.ClusterDomain, "cluster-domain", "cluster.local", "The cluster domain ending that should be used for the virtual cluster") - cmd.Flags().StringVar(&options.ServiceCIDR, "service-cidr", "10.96.0.0/12", "Service CIDR is the subnet used by k8s services") + cmd.Flags().StringVar(&options.ServiceCIDR, "service-cidr", "", "Service CIDR is the subnet used by k8s services") cmd.Flags().StringVar(&options.Prefix, "prefix", "vcluster", "Release name and prefix for generating the assets") cmd.Flags().StringVar(&options.Namespace, "namespace", "", "Namespace where to deploy the cert secret to") cmd.Flags().StringVar(&options.CertificateDir, "certificate-dir", "certs", "The temporary directory where the certificates will be stored") @@ -105,6 +107,15 @@ func ExecuteCerts(options *CertsCmd) error { } } + cidr := options.ServiceCIDR + if cidr == "" { + cidr, err = servicecidr.EnsureServiceCIDRConfigmap(context.Background(), kubeClient, options.Namespace, options.Prefix) + if err != nil { + klog.Errorf("Failed to retrieve service CIDR range") + return err + } + } + secretName := options.Prefix + "-certs" _, err = kubeClient.CoreV1().Secrets(options.Namespace).Get(context.Background(), secretName, metav1.GetOptions{}) if err == nil { @@ -131,7 +142,7 @@ func ExecuteCerts(options *CertsCmd) error { ServerCertSANs: serverSans, PeerCertSANs: serverSans, } - cfg.Networking.ServiceSubnet = options.ServiceCIDR + cfg.Networking.ServiceSubnet = cidr cfg.Networking.DNSDomain = options.ClusterDomain cfg.ControlPlaneEndpoint = options.Prefix + "-api" cfg.CertificatesDir = options.CertificateDir diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index 4f4f46cd66..8ddba0a193 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -3,20 +3,11 @@ package cmd import ( "context" "fmt" - "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" - "github.com/loft-sh/vcluster/pkg/util/pluginhookclient" "io/ioutil" "math" "os" "time" - kerrors "k8s.io/apimachinery/pkg/api/errors" - - corev1 "k8s.io/api/core/v1" - - "github.com/loft-sh/vcluster/pkg/plugin" - - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" context2 "github.com/loft-sh/vcluster/cmd/vcluster/context" "github.com/loft-sh/vcluster/pkg/apis" "github.com/loft-sh/vcluster/pkg/controllers" @@ -26,16 +17,23 @@ import ( "github.com/loft-sh/vcluster/pkg/controllers/resources/services" "github.com/loft-sh/vcluster/pkg/coredns" "github.com/loft-sh/vcluster/pkg/leaderelection" - + "github.com/loft-sh/vcluster/pkg/plugin" "github.com/loft-sh/vcluster/pkg/server" + "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" "github.com/loft-sh/vcluster/pkg/util/clienthelper" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" + "github.com/loft-sh/vcluster/pkg/util/pluginhookclient" + "github.com/loft-sh/vcluster/pkg/util/servicecidr" "github.com/loft-sh/vcluster/pkg/util/toleration" "github.com/loft-sh/vcluster/pkg/util/translate" + + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -159,9 +157,48 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error { return fmt.Errorf("invalid argument enforce-pod-security-standard=%s, must be one of: privileged, baseline, restricted", options.EnforcePodSecurityStandard) } + // set suffix + translate.Suffix = options.Name + if translate.Suffix == "" { + translate.Suffix = options.DeprecatedSuffix + } + if translate.Suffix == "" { + translate.Suffix = "vcluster" + } + + // get current namespace + currentNamespace, err := clienthelper.CurrentNamespace() + if err != nil { + return err + } + + // get host cluster config and tweak rate-limiting configuration + inClusterConfig := ctrl.GetConfigOrDie() + inClusterConfig.QPS = 40 + inClusterConfig.Burst = 80 + inClusterConfig.Timeout = 0 + + inClusterClient, err := kubernetes.NewForConfig(inClusterConfig) + if err != nil { + return err + } + + // Ensure that service CIDR range is written into the expected location + err = wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) { + err = ensureServiceCIDR(inClusterClient, currentNamespace, translate.Suffix) + if err != nil { + klog.Errorf("failed to ensure that service CIDR range is written into the expected location: %v", err) + return false, nil + } + return true, nil + }) + if err != nil { + return err + } + // wait until kube config is available var clientConfig clientcmd.ClientConfig - err := wait.Poll(time.Second, time.Hour, func() (bool, error) { + err = wait.Poll(time.Second, time.Hour, func() (bool, error) { out, err := ioutil.ReadFile(options.KubeConfig) if err != nil { if os.IsNotExist(err) { @@ -213,15 +250,6 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error { } } - // set suffix - translate.Suffix = options.Name - if translate.Suffix == "" { - translate.Suffix = options.DeprecatedSuffix - } - if translate.Suffix == "" { - translate.Suffix = "vcluster" - } - // check if enable scheduler works correctly if options.EnableScheduler && !options.SyncAllNodes && len(options.NodeSelector) == 0 { options.SyncAllNodes = true @@ -240,12 +268,6 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error { // set kubelet port nodeservice.KubeletTargetPort = options.Port - // get current namespace - currentNamespace, err := clienthelper.CurrentNamespace() - if err != nil { - return err - } - // ensure target namespace if options.TargetNamespace == "" { options.TargetNamespace = currentNamespace @@ -255,17 +277,12 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error { if err != nil { return err } - inClusterConfig := ctrl.GetConfigOrDie() // We increase the limits here so that we don't get any problems virtualClusterConfig.QPS = 1000 virtualClusterConfig.Burst = 2000 virtualClusterConfig.Timeout = 0 - inClusterConfig.QPS = 40 - inClusterConfig.Burst = 80 - inClusterConfig.Timeout = 0 - // start leader election for controllers rawConfig, err := clientConfig.RawConfig() if err != nil { @@ -354,6 +371,24 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error { return nil } +func ensureServiceCIDR(c kubernetes.Interface, currentNamespace, vclusterName string) error { + // check if k0s config Secret exists + _, err := c.CoreV1().Secrets(currentNamespace).Get(context.Background(), servicecidr.GetK0sSecretName(vclusterName), metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return err + } + + // if k0s secret was found ensure it contains service CIDR range + if err == nil { + klog.Info("k0s config secret detected, syncer will ensure that it contains service CIDR") + return servicecidr.EnsureServiceCIDRInK0sSecret(context.Background(), c, currentNamespace, vclusterName) + } + + // in all other cases ensure that a valid CIDR range is in the designated ConfigMap + _, err = servicecidr.EnsureServiceCIDRConfigmap(context.Background(), c, currentNamespace, vclusterName) + return err +} + func startControllers(ctx *context2.ControllerContext, rawConfig *api.Config, serverVersion *version.Info) error { // setup CoreDNS according to the manifest file go func() { diff --git a/cmd/vclusterctl/cmd/create.go b/cmd/vclusterctl/cmd/create.go index 4645e8a328..1b41c8e993 100644 --- a/cmd/vclusterctl/cmd/create.go +++ b/cmd/vclusterctl/cmd/create.go @@ -13,6 +13,7 @@ import ( "github.com/loft-sh/vcluster/pkg/helm/values" "github.com/loft-sh/vcluster/pkg/upgrade" "github.com/loft-sh/vcluster/pkg/util" + "github.com/loft-sh/vcluster/pkg/util/servicecidr" "golang.org/x/mod/semver" "github.com/loft-sh/vcluster/cmd/vclusterctl/flags" @@ -168,7 +169,7 @@ func (cmd *CreateCmd) Run(args []string) error { // get service cidr if cmd.CIDR == "" { - cmd.CIDR = values.GetServiceCIDR(client, cmd.Namespace) + cmd.CIDR = servicecidr.GetServiceCIDR(client, cmd.Namespace) } var kubernetesVersion *version.Info diff --git a/cmd/vclusterctl/cmd/get/service_cidr.go b/cmd/vclusterctl/cmd/get/service_cidr.go index da65a77e23..0058719163 100644 --- a/cmd/vclusterctl/cmd/get/service_cidr.go +++ b/cmd/vclusterctl/cmd/get/service_cidr.go @@ -5,7 +5,7 @@ import ( "github.com/loft-sh/vcluster/cmd/vclusterctl/flags" "github.com/loft-sh/vcluster/cmd/vclusterctl/log" - "github.com/loft-sh/vcluster/pkg/helm/values" + "github.com/loft-sh/vcluster/pkg/util/servicecidr" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -69,7 +69,7 @@ func (cmd *serviceCIDRCmd) Run(cobraCmd *cobra.Command) error { } } - cidr := values.GetServiceCIDR(client, cmd.Namespace) + cidr := servicecidr.GetServiceCIDR(client, cmd.Namespace) _, err = cmd.log.Write([]byte(cidr)) if err != nil { diff --git a/devspace.yaml b/devspace.yaml index 9f784331f6..00a17c4fb6 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -4,10 +4,6 @@ vars: value: ghcr.io/loft-sh/loft-enterprise/dev-vcluster - name: K3S_IMAGE value: rancher/k3s:v1.23.5-k3s1 - # Replace this with your clusters service CIDR, you can find it out via - # kubectl apply -f hack/wrong-cluster-ip-service.yaml - - name: SERVICE_CIDR - value: 10.96.0.0/12 images: vcluster: image: ${SYNCER_IMAGE} @@ -22,7 +18,7 @@ deployments: chart: name: ./charts/k3s values: - serviceCIDR: ${SERVICE_CIDR} + serviceCIDR: $([ $1 == "dev" ] && vcluster get service-cidr || echo "null") mapServices: fromVirtual: - from: test/test diff --git a/pkg/helm/values/k3s.go b/pkg/helm/values/k3s.go index 5938579e3b..90702b0e25 100644 --- a/pkg/helm/values/k3s.go +++ b/pkg/helm/values/k3s.go @@ -1,7 +1,6 @@ package values import ( - "context" "fmt" "regexp" "strconv" @@ -9,15 +8,7 @@ import ( "github.com/loft-sh/vcluster/pkg/helm" "github.com/loft-sh/vcluster/pkg/log" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/kubernetes" -) - -var ( - errorMessageIPFamily = "expected an IPv6 value as indicated by " // Dual-stack cluster with .spec.ipFamilies=["IPv6"] - errorMessageIPv4Disabled = "IPv4 is not configured on this cluster" // IPv6 only cluster ) var K3SVersionMap = map[string]string{ @@ -48,7 +39,6 @@ var baseArgsMap = map[string]string{ } var replaceRegEx = regexp.MustCompile("[^0-9]+") -var errorMessageFind = "The range of valid IPs is " func getDefaultK3SReleaseValues(chartOptions *helm.ChartOptions, log log.Logger) (string, error) { var ( @@ -161,50 +151,3 @@ func GetKubernetesVersion(serverVersion *version.Info) string { func GetKubernetesMinorVersion(serverVersion *version.Info) (int, error) { return strconv.Atoi(replaceRegEx.ReplaceAllString(serverVersion.Minor, "")) } - -func getServiceCIDR(client kubernetes.Interface, namespace string, ipv6 bool) (string, error) { - clusterIP := "4.4.4.4" - if ipv6 { - // https://www.ietf.org/rfc/rfc3849.txt - clusterIP = "2001:DB8::1" - } - _, err := client.CoreV1().Services(namespace).Create(context.Background(), &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-service-", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Port: 80, - }, - }, - ClusterIP: clusterIP, - }, - }, metav1.CreateOptions{}) - if err == nil { - return "", fmt.Errorf("couldn't find cluster service cidr, will fallback to 10.96.0.0/12, however this is probably wrong, please make sure the host cluster service cidr and virtual cluster service cidr match") - } - - errorMessage := err.Error() - idx := strings.Index(errorMessage, errorMessageFind) - if idx == -1 { - return "", fmt.Errorf("couldn't find cluster service cidr (" + errorMessage + "), will fallback to 10.96.0.0/12, however this is probably wrong, please make sure the host cluster service cidr and virtual cluster service cidr match") - } - - return strings.TrimSpace(errorMessage[idx+len(errorMessageFind):]), nil -} - -func GetServiceCIDR(client kubernetes.Interface, namespace string) string { - cidr, err := getServiceCIDR(client, namespace, false) - if err != nil { - idx := strings.Index(err.Error(), errorMessageIPFamily) - idz := strings.Index(err.Error(), errorMessageIPv4Disabled) - if idx != -1 || idz != -1 { - cidr, err = getServiceCIDR(client, namespace, true) - } - if err != nil { - return "10.96.0.0/12" - } - } - return cidr -} diff --git a/pkg/util/servicecidr/servicecidr.go b/pkg/util/servicecidr/servicecidr.go new file mode 100644 index 0000000000..9560af5c49 --- /dev/null +++ b/pkg/util/servicecidr/servicecidr.go @@ -0,0 +1,159 @@ +package servicecidr + +import ( + "context" + "fmt" + "net" + "strings" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + CIDRConfigMapPrefix = "vc-cidr-" + CIDRConfigMapKey = "cidr" + K0sConfigKey = "config.yaml" + K0sCIDRPlaceHolder = "CIDR_PLACEHOLDER" + K0sConfigReadyFlag = "CONFIG_READY" + + ErrorMessageIPFamily = "expected an IPv6 value as indicated by " // Dual-stack cluster with .spec.ipFamilies=["IPv6"] + ErrorMessageIPv4Disabled = "IPv4 is not configured on this cluster" // IPv6 only cluster + ErrorMessageFind = "The range of valid IPs is " +) + +func GetCIDRConfigMapName(vclusterName string) string { + return fmt.Sprintf("%s%s", CIDRConfigMapPrefix, vclusterName) +} + +func GetK0sSecretName(vclusterName string) string { + return fmt.Sprintf("vc-%s-config", vclusterName) +} + +func EnsureServiceCIDRConfigmap(ctx context.Context, c kubernetes.Interface, currentNamespace string, vclusterName string) (string, error) { + cm, err := c.CoreV1().ConfigMaps(currentNamespace).Get(ctx, GetCIDRConfigMapName(vclusterName), metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return "", err + } + exists := !kerrors.IsNotFound(err) + if !exists { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetCIDRConfigMapName(vclusterName), + Namespace: currentNamespace, + }, + } + } + cidrData, ok := cm.Data[CIDRConfigMapKey] + // do nothing if a valid CIDR is already present in the expected Configmap data key + if exists && ok { + _, _, err = net.ParseCIDR(cidrData) + if err == nil { + return cidrData, err + } + } + + // find out correct cidr + cidr := GetServiceCIDR(c, currentNamespace) + + if !exists { + cm.Data = map[string]string{ + CIDRConfigMapKey: cidr, + } + _, err = c.CoreV1().ConfigMaps(currentNamespace).Create(ctx, cm, metav1.CreateOptions{}) + return cidr, err + } + + // create and execute a Patch call for the ConfigMap + originalObject := cm.DeepCopy() + patch := client.MergeFrom(originalObject) + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data[CIDRConfigMapKey] = cidr + data, err := patch.Data(cm) + if err != nil { + return "", fmt.Errorf("failed to create patch for the %s/%s Configmap: %v", cm.Namespace, cm.Name, err) + } + _, err = c.CoreV1().ConfigMaps(currentNamespace).Patch(ctx, cm.Name, patch.Type(), data, metav1.PatchOptions{}) + return cidr, err +} + +func EnsureServiceCIDRInK0sSecret(ctx context.Context, c kubernetes.Interface, currentNamespace string, vclusterName string) error { + secret, err := c.CoreV1().Secrets(currentNamespace).Get(context.Background(), GetK0sSecretName(vclusterName), metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not read k0s configuration secret %s/%s: %v", currentNamespace, GetK0sSecretName(vclusterName), err) + } + configData, ok := secret.Data[K0sConfigKey] + if !ok { + return fmt.Errorf("k0s configuration secret %s/%s does not contain the expected key - %s", secret.Namespace, secret.Name, K0sConfigKey) + } + + // find out correct cidr + cidr := GetServiceCIDR(c, currentNamespace) + newData := strings.ReplaceAll(string(configData), K0sCIDRPlaceHolder, cidr) + + originalObject := secret.DeepCopy() + secret.Data[K0sConfigKey] = []byte(newData) + secret.Data[K0sConfigReadyFlag] = []byte("true") + patch := client.MergeFrom(originalObject) + data, err := patch.Data(secret) + if err != nil { + return fmt.Errorf("failed to create patch for the %s/%s Secret: %v", secret.Namespace, secret.Name, err) + } + _, err = c.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, patch.Type(), data, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("failed to patch k0s configuration secret %s/%s: %v", secret.Namespace, secret.Name, err) + } + return nil +} + +func GetServiceCIDR(client kubernetes.Interface, namespace string) string { + cidr, err := getServiceCIDR(client, namespace, false) + if err != nil { + idx := strings.Index(err.Error(), ErrorMessageIPFamily) + idz := strings.Index(err.Error(), ErrorMessageIPv4Disabled) + if idx != -1 || idz != -1 { + cidr, err = getServiceCIDR(client, namespace, true) + } + if err != nil { + return "10.96.0.0/12" + } + } + return cidr +} + +func getServiceCIDR(client kubernetes.Interface, namespace string, ipv6 bool) (string, error) { + clusterIP := "4.4.4.4" + if ipv6 { + // https://www.ietf.org/rfc/rfc3849.txt + clusterIP = "2001:DB8::1" + } + _, err := client.CoreV1().Services(namespace).Create(context.Background(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-service-", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: 80, + }, + }, + ClusterIP: clusterIP, + }, + }, metav1.CreateOptions{}) + if err == nil { + return "", fmt.Errorf("couldn't find cluster service cidr, will fallback to 10.96.0.0/12, however this is probably wrong, please make sure the host cluster service cidr and virtual cluster service cidr match") + } + + errorMessage := err.Error() + idx := strings.Index(errorMessage, ErrorMessageFind) + if idx == -1 { + return "", fmt.Errorf("couldn't find cluster service cidr (" + errorMessage + "), will fallback to 10.96.0.0/12, however this is probably wrong, please make sure the host cluster service cidr and virtual cluster service cidr match") + } + + return strings.TrimSpace(errorMessage[idx+len(ErrorMessageFind):]), nil +}