From 8f589f24f48228291f35b0a61d54caaa06708608 Mon Sep 17 00:00:00 2001 From: fabiankramm Date: Fri, 7 May 2021 09:21:12 +0200 Subject: [PATCH] feat: support for kubernetes v1.21.0 --- chart/values.yaml | 2 +- cmd/vclusterctl/cmd/create.go | 9 +- cmd/vclusterctl/cmd/list.go | 105 ++++++++++++++++++ cmd/vclusterctl/cmd/root.go | 1 + pkg/constants/indices.go | 11 +- pkg/controllers/resources/pods/syncer.go | 2 +- pkg/controllers/resources/pods/translate.go | 91 ++++++++++++++- pkg/controllers/resources/pods/util.go | 23 +++- pkg/controllers/resources/secrets/syncer.go | 44 ++++---- .../resources/secrets/syncer_test.go | 8 +- 10 files changed, 256 insertions(+), 40 deletions(-) create mode 100644 cmd/vclusterctl/cmd/list.go diff --git a/chart/values.yaml b/chart/values.yaml index 6da68bdb6..02d526b64 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -10,7 +10,7 @@ syncer: resources: {} vcluster: - image: rancher/k3s:v1.19.1-k3s1 + image: rancher/k3s:v1.21.0-k3s1 command: - /bin/k3s baseArgs: diff --git a/cmd/vclusterctl/cmd/create.go b/cmd/vclusterctl/cmd/create.go index 86ce5a48a..c9f1e6dfd 100644 --- a/cmd/vclusterctl/cmd/create.go +++ b/cmd/vclusterctl/cmd/create.go @@ -22,6 +22,7 @@ import ( ) var VersionMap = map[string]string{ + "1.21": "rancher/k3s:v1.21.0-k3s1", "1.20": "rancher/k3s:v1.20.4-k3s1", "1.19": "rancher/k3s:v1.19.8-k3s1", "1.18": "rancher/k3s:v1.18.16-k3s1", @@ -208,10 +209,10 @@ func getReleaseValues(client kubernetes.Interface, namespace string, disableIngr image, ok := VersionMap[serverVersionString] if !ok { - if serverMinorInt > 20 { - log.Infof("officially unsupported host server version %s, will fallback to virtual cluster version v1.20", serverVersionString) - image = VersionMap["1.20"] - serverVersionString = "1.20" + if serverMinorInt > 21 { + log.Infof("officially unsupported host server version %s, will fallback to virtual cluster version v1.21", serverVersionString) + image = VersionMap["1.21"] + serverVersionString = "1.21" } else { log.Infof("officially unsupported host server version %s, will fallback to virtual cluster version v1.16", serverVersionString) image = VersionMap["1.16"] diff --git a/cmd/vclusterctl/cmd/list.go b/cmd/vclusterctl/cmd/list.go new file mode 100644 index 000000000..6cc7cbd48 --- /dev/null +++ b/cmd/vclusterctl/cmd/list.go @@ -0,0 +1,105 @@ +package cmd + +import ( + "context" + "github.com/loft-sh/vcluster/cmd/vclusterctl/flags" + "github.com/loft-sh/vcluster/cmd/vclusterctl/log" + "github.com/pkg/errors" + "github.com/spf13/cobra" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// ListCmd holds the login cmd flags +type ListCmd struct { + *flags.GlobalFlags + + Namespace string + log log.Logger +} + +// NewListCmd creates a new command +func NewListCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &ListCmd{ + GlobalFlags: globalFlags, + log: log.GetInstance(), + } + + cobraCmd := &cobra.Command{ + Use: "list", + Short: "Lists all virtual clusters", + Long: ` +####################################################### +#################### vcluster list #################### +####################################################### +Lists all virtual clusters + +Example: +vcluster list +vcluster list --namespace test +####################################################### + `, + Args: cobra.NoArgs, + RunE: func(cobraCmd *cobra.Command, args []string) error { + return cmd.Run(cobraCmd, args) + }, + } + + cobraCmd.Flags().StringVarP(&cmd.Namespace, "namespace", "n", "", "The namespace the vcluster was created in") + return cobraCmd +} + +// Run executes the functionality +func (cmd *ListCmd) Run(cobraCmd *cobra.Command, args []string) error { + // first load the kube config + kubeClientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + namespace := metav1.NamespaceAll + if cmd.Namespace != "" { + namespace = cmd.Namespace + } + + // get all statefulsets with the label app=vcluster + restConfig, err := kubeClientConfig.ClientConfig() + if err != nil { + return err + } + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + + statefulSets, err := client.AppsV1().StatefulSets(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "app=vcluster"}) + if err != nil { + if kerrors.IsForbidden(err) { + // try the current namespace instead + namespace, _, err = kubeClientConfig.Namespace() + if err != nil { + return err + } else if namespace == "" { + namespace = "default" + } + + statefulSets, err = client.AppsV1().StatefulSets(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "app=vcluster"}) + if err != nil { + return err + } + } else { + return errors.Wrap(err, "list stateful sets") + } + } + + header := []string{"NAME", "NAMESPACE", "CREATED"} + values := [][]string{} + for _, s := range statefulSets.Items { + values = append(values, []string{ + s.Name, + s.Namespace, + s.CreationTimestamp.String(), + }) + } + + log.PrintTable(cmd.log, header, values) + return nil +} diff --git a/cmd/vclusterctl/cmd/root.go b/cmd/vclusterctl/cmd/root.go index 3836c6689..831f1dbc8 100644 --- a/cmd/vclusterctl/cmd/root.go +++ b/cmd/vclusterctl/cmd/root.go @@ -51,6 +51,7 @@ func BuildRoot(log log.Logger) *cobra.Command { // add top level commands rootCmd.AddCommand(NewConnectCmd(globalFlags)) rootCmd.AddCommand(NewCreateCmd(globalFlags)) + rootCmd.AddCommand(NewListCmd(globalFlags)) rootCmd.AddCommand(NewDeleteCmd(globalFlags)) return rootCmd diff --git a/pkg/constants/indices.go b/pkg/constants/indices.go index 8a68458a8..c6da47664 100644 --- a/pkg/constants/indices.go +++ b/pkg/constants/indices.go @@ -1,10 +1,9 @@ package constants const ( - IndexByVName = "IndexByVName" - IndexByAssigned = "IndexByAssigned" - IndexByStorageClass = "IndexByStorageClass" - - IndexBySecret = "IndexBySecret" - IndexByConfigMap = "IndexByConfigMap" + IndexByVName = "IndexByVName" + IndexByAssigned = "IndexByAssigned" + IndexByStorageClass = "IndexByStorageClass" + IndexByIngressSecret = "IndexByIngressSecret" + IndexByConfigMap = "IndexByConfigMap" ) diff --git a/pkg/controllers/resources/pods/syncer.go b/pkg/controllers/resources/pods/syncer.go index a1e63af7b..3c1dec3c5 100644 --- a/pkg/controllers/resources/pods/syncer.go +++ b/pkg/controllers/resources/pods/syncer.go @@ -261,7 +261,7 @@ func (s *syncer) translatePod(vPod *corev1.Pod, pPod *corev1.Pod) error { ptrServiceList = append(ptrServiceList, &s) } - return translatePod(pPod, vPod, ptrServiceList, s.clusterDomain, dnsIP, kubeIP, s.serviceAccountName, s.translateImages, s.overrideHosts, s.overrideHostsImage) + return translatePod(pPod, vPod, s.virtualClient, ptrServiceList, s.clusterDomain, dnsIP, kubeIP, s.serviceAccountName, s.translateImages, s.overrideHosts, s.overrideHostsImage) } func (s *syncer) findKubernetesIP() (string, error) { diff --git a/pkg/controllers/resources/pods/translate.go b/pkg/controllers/resources/pods/translate.go index 9814d9964..8553fb699 100644 --- a/pkg/controllers/resources/pods/translate.go +++ b/pkg/controllers/resources/pods/translate.go @@ -1,7 +1,10 @@ package pods import ( + "context" "fmt" + "github.com/pkg/errors" + "sigs.k8s.io/controller-runtime/pkg/client" "sort" "strings" @@ -25,7 +28,7 @@ const ( HostsRewriteContainerName = "vcluster-rewrite-hosts" ) -func translatePod(pPod *corev1.Pod, vPod *corev1.Pod, services []*corev1.Service, clusterDomain, dnsIP, kubeIP, serviceAccount string, translator ImageTranslator, enableOverrideHosts bool, overrideHostsImage string) error { +func translatePod(pPod *corev1.Pod, vPod *corev1.Pod, vClient client.Client, services []*corev1.Service, clusterDomain, dnsIP, kubeIP, serviceAccount string, translator ImageTranslator, enableOverrideHosts bool, overrideHostsImage string) error { pPod.Status = corev1.PodStatus{} pPod.Spec.DeprecatedServiceAccount = "" pPod.Spec.ServiceAccountName = serviceAccount @@ -188,6 +191,13 @@ func translatePod(pPod *corev1.Pod, vPod *corev1.Pod, services []*corev1.Service if pPod.Spec.Volumes[i].PersistentVolumeClaim != nil { pPod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName = translate.PhysicalName(pPod.Spec.Volumes[i].PersistentVolumeClaim.ClaimName, vPod.Namespace) } + if pPod.Spec.Volumes[i].Projected != nil { + // get old service account name + err := translateProjectedVolume(pPod.Spec.Volumes[i].Projected, vClient, vPod) + if err != nil { + return err + } + } } // we add an annotation if the pod has a replica set or statefulset owner @@ -205,6 +215,85 @@ func translatePod(pPod *corev1.Pod, vPod *corev1.Pod, services []*corev1.Service return nil } +func secretNameFromServiceAccount(vClient client.Client, vPod *corev1.Pod) (string, error) { + vServiceAccount := "" + if vPod.Spec.ServiceAccountName != "" { + vServiceAccount = vPod.Spec.ServiceAccountName + } else if vPod.Spec.DeprecatedServiceAccount != "" { + vServiceAccount = vPod.Spec.DeprecatedServiceAccount + } + + secretList := &corev1.SecretList{} + err := vClient.List(context.Background(), secretList, client.InNamespace(vPod.Namespace)) + if err != nil { + return "", errors.Wrap(err, "list secrets in "+vPod.Namespace) + } + for _, secret := range secretList.Items { + if secret.Annotations["kubernetes.io/service-account.name"] == vServiceAccount { + return secret.Name, nil + } + } + + return "", nil +} + +func translateProjectedVolume(projectedVolume *corev1.ProjectedVolumeSource, vClient client.Client, vPod *corev1.Pod) error { + for i := range projectedVolume.Sources { + if projectedVolume.Sources[i].Secret != nil { + projectedVolume.Sources[i].Secret.Name = translate.PhysicalName(projectedVolume.Sources[i].Secret.Name, vPod.Namespace) + } + if projectedVolume.Sources[i].ConfigMap != nil { + projectedVolume.Sources[i].ConfigMap.Name = translate.PhysicalName(projectedVolume.Sources[i].ConfigMap.Name, vPod.Namespace) + } + if projectedVolume.Sources[i].DownwardAPI != nil { + for j := range projectedVolume.Sources[i].DownwardAPI.Items { + translateFieldRef(projectedVolume.Sources[i].DownwardAPI.Items[j].FieldRef) + } + } + if projectedVolume.Sources[i].ServiceAccountToken != nil { + secretName, err := secretNameFromServiceAccount(vClient, vPod) + if err != nil { + return err + } else if secretName == "" { + return fmt.Errorf("couldn't find service account secret for pod %s/%s", vPod.Namespace, vPod.Name) + } + + allRights := int32(0644) + projectedVolume.Sources[i].Secret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: translate.PhysicalName(secretName, vPod.Namespace), + }, + Items: []corev1.KeyToPath{ + { + Path: projectedVolume.Sources[i].ServiceAccountToken.Path, + Key: "token", + Mode: &allRights, + }, + }, + } + projectedVolume.Sources[i].ServiceAccountToken = nil + } + } + + return nil +} + +func translateFieldRef(fieldSelector *corev1.ObjectFieldSelector) { + if fieldSelector == nil { + return + } + switch fieldSelector.FieldPath { + case "metadata.name": + fieldSelector.FieldPath = "metadata.annotations['" + NameAnnotation + "']" + case "metadata.namespace": + fieldSelector.FieldPath = "metadata.annotations['" + NamespaceAnnotation + "']" + case "metadata.uid": + fieldSelector.FieldPath = "metadata.annotations['" + UIDAnnotation + "']" + case "spec.serviceAccountName": + fieldSelector.FieldPath = "metadata.annotations['" + ServiceAccountNameAnnotation + "']" + } +} + func stripHostRewriteContainer(pPod *corev1.Pod) *corev1.Pod { if pPod.Annotations == nil || pPod.Annotations[HostsRewrittenAnnotation] != "true" { return pPod diff --git a/pkg/controllers/resources/pods/util.go b/pkg/controllers/resources/pods/util.go index 7ea543f5d..5c9ab3ede 100644 --- a/pkg/controllers/resources/pods/util.go +++ b/pkg/controllers/resources/pods/util.go @@ -3,6 +3,7 @@ package pods import ( "github.com/loft-sh/vcluster/pkg/util/translate" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func ConfigNamesFromPod(pod *corev1.Pod) []string { @@ -20,11 +21,18 @@ func ConfigNamesFromPod(pod *corev1.Pod) []string { if pod.Spec.Volumes[i].ConfigMap != nil { configMaps = append(configMaps, pod.Namespace+"/"+pod.Spec.Volumes[i].ConfigMap.Name) } + if pod.Spec.Volumes[i].Projected != nil { + for j := range pod.Spec.Volumes[i].Projected.Sources { + if pod.Spec.Volumes[i].Projected.Sources[j].ConfigMap != nil { + configMaps = append(configMaps, pod.Namespace+"/"+pod.Spec.Volumes[i].Projected.Sources[j].ConfigMap.Name) + } + } + } } return translate.UniqueSlice(configMaps) } -func SecretNamesFromPod(pod *corev1.Pod) []string { +func SecretNamesFromPod(vClient client.Client, pod *corev1.Pod) []string { secrets := []string{} for _, c := range pod.Spec.Containers { secrets = append(secrets, SecretNamesFromContainer(pod.Namespace, &c)...) @@ -42,6 +50,19 @@ func SecretNamesFromPod(pod *corev1.Pod) []string { if pod.Spec.Volumes[i].Secret != nil { secrets = append(secrets, pod.Namespace+"/"+pod.Spec.Volumes[i].Secret.SecretName) } + if pod.Spec.Volumes[i].Projected != nil { + for j := range pod.Spec.Volumes[i].Projected.Sources { + if pod.Spec.Volumes[i].Projected.Sources[j].Secret != nil { + secrets = append(secrets, pod.Namespace+"/"+pod.Spec.Volumes[i].Projected.Sources[j].Secret.Name) + } + if pod.Spec.Volumes[i].Projected.Sources[j].ServiceAccountToken != nil { + secretName, err := secretNameFromServiceAccount(vClient, pod) + if err == nil && secretName != "" { + secrets = append(secrets, pod.Namespace+"/"+secretName) + } + } + } + } } return translate.UniqueSlice(secrets) } diff --git a/pkg/controllers/resources/secrets/syncer.go b/pkg/controllers/resources/secrets/syncer.go index ead3a92df..716885e9a 100644 --- a/pkg/controllers/resources/secrets/syncer.go +++ b/pkg/controllers/resources/secrets/syncer.go @@ -30,21 +30,27 @@ import ( "time" ) -func indexPodBySecret(rawObj client.Object) []string { - pod := rawObj.(*corev1.Pod) - return pods.SecretNamesFromPod(pod) -} - -func Register(ctx *context2.ControllerContext) error { - // index pods by their used secrets - err := ctx.VirtualManager.GetFieldIndexer().IndexField(ctx.Context, &corev1.Pod{}, constants.IndexBySecret, indexPodBySecret) +func isSecretUsedByPods(ctx context.Context, vClient client.Client, secretName string) (bool, error) { + podList := &corev1.PodList{} + err := vClient.List(ctx, podList) if err != nil { - return err + return false, err + } + for _, pod := range podList.Items { + for _, secret := range pods.SecretNamesFromPod(vClient, &pod) { + if secret == secretName { + return true, nil + } + } } + return false, nil +} + +func Register(ctx *context2.ControllerContext) error { includeIngresses := strings.Contains(ctx.Options.DisableSyncResources, "ingresses") == false if includeIngresses { - err := ctx.VirtualManager.GetFieldIndexer().IndexField(ctx.Context, &networkingv1beta1.Ingress{}, constants.IndexBySecret, func(rawObj client.Object) []string { + err := ctx.VirtualManager.GetFieldIndexer().IndexField(ctx.Context, &networkingv1beta1.Ingress{}, constants.IndexByIngressSecret, func(rawObj client.Object) []string { ingress := rawObj.(*networkingv1beta1.Ingress) return ingresses.SecretNamesFromIngress(ingress) }) @@ -68,7 +74,9 @@ func Register(ctx *context2.ControllerContext) error { builder = builder.Watches(&source.Kind{Type: &networkingv1beta1.Ingress{}}, handler.EnqueueRequestsFromMapFunc(mapIngresses)) } - return builder.Watches(&source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc(mapPods)) + return builder.Watches(&source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request { + return mapPods(object, ctx.VirtualManager.GetClient()) + })) }, }) } @@ -96,14 +104,14 @@ func mapIngresses(obj client.Object) []reconcile.Request { return requests } -func mapPods(obj client.Object) []reconcile.Request { +func mapPods(obj client.Object, vClient client.Client) []reconcile.Request { pod, ok := obj.(*corev1.Pod) if !ok { return nil } requests := []reconcile.Request{} - names := pods.SecretNamesFromPod(pod) + names := pods.SecretNamesFromPod(vClient, pod) for _, name := range names { splitted := strings.Split(name, "/") if len(splitted) == 2 { @@ -185,20 +193,18 @@ func (s *syncer) isSecretUsed(vObj runtime.Object) (bool, error) { return false, fmt.Errorf("%#v is not a secret", vObj) } - podList := &corev1.PodList{} - err := s.virtualClient.List(context.TODO(), podList, client.MatchingFields{constants.IndexBySecret: secret.Namespace + "/" + secret.Name}) + isUsed, err := isSecretUsedByPods(context.TODO(), s.virtualClient, secret.Namespace+"/"+secret.Name) if err != nil { - return false, err + return false, errors.Wrap(err, "is secret used by pods") } - - if len(podList.Items) > 0 { + if isUsed { return true, nil } // check if we also sync ingresses if s.includeIngresses { ingressesList := &networkingv1beta1.IngressList{} - err := s.virtualClient.List(context.TODO(), ingressesList, client.MatchingFields{constants.IndexBySecret: secret.Namespace + "/" + secret.Name}) + err := s.virtualClient.List(context.TODO(), ingressesList, client.MatchingFields{constants.IndexByIngressSecret: secret.Namespace + "/" + secret.Name}) if err != nil { return false, err } diff --git a/pkg/controllers/resources/secrets/syncer_test.go b/pkg/controllers/resources/secrets/syncer_test.go index fbe23914f..9f52cbf43 100644 --- a/pkg/controllers/resources/secrets/syncer_test.go +++ b/pkg/controllers/resources/secrets/syncer_test.go @@ -2,7 +2,6 @@ package secrets import ( "context" - "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/util/loghelper" testingutil "github.com/loft-sh/vcluster/pkg/util/testing" "github.com/loft-sh/vcluster/pkg/util/translate" @@ -17,11 +16,6 @@ import ( ) func newFakeSyncer(ctx context.Context, pClient *testingutil.FakeIndexClient, vClient *testingutil.FakeIndexClient) *syncer { - err := vClient.IndexField(ctx, &corev1.Pod{}, constants.IndexBySecret, indexPodBySecret) - if err != nil { - panic(err) - } - return &syncer{ eventRecoder: &testingutil.FakeEventRecorder{}, targetNamespace: "test", @@ -242,7 +236,7 @@ func TestMapping(t *testing.T) { }, }, } - requests = mapPods(pod) + requests = mapPods(pod, testingutil.NewFakeClient(testingutil.NewScheme())) if len(requests) != 2 || requests[0].Name != "a" || requests[0].Namespace != "test" || requests[1].Name != "b" || requests[1].Namespace != "test" { t.Fatalf("Wrong pod requests returned: %#+v", requests) }