From 27ef31ba852fa04ee14649894791651d6fe7da90 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Fri, 22 Sep 2023 12:05:51 +0200 Subject: [PATCH] feat: allow vcluster pro via vcluster create & delete --- cmd/vclusterctl/cmd/app/create/pro.go | 178 ++++++++++++++++++ cmd/vclusterctl/cmd/app/create/types.go | 23 ++- cmd/vclusterctl/cmd/create.go | 98 +++++++--- cmd/vclusterctl/cmd/delete.go | 118 ++++++++++-- cmd/vclusterctl/cmd/find/find.go | 50 +++-- go.mod | 2 +- go.sum | 2 + pkg/helm/helm.go | 35 +--- pkg/pro/create.go | 1 + .../v3/cmd/loftctl/cmd/create/space.go | 2 +- .../v3/cmd/loftctl/cmd/create/vcluster.go | 52 +++-- .../loftctl/v3/pkg/client/helper/helper.go | 4 +- vendor/modules.txt | 2 +- 13 files changed, 464 insertions(+), 103 deletions(-) create mode 100644 cmd/vclusterctl/cmd/app/create/pro.go create mode 100644 pkg/pro/create.go diff --git a/cmd/vclusterctl/cmd/app/create/pro.go b/cmd/vclusterctl/cmd/app/create/pro.go new file mode 100644 index 0000000000..be497618c0 --- /dev/null +++ b/cmd/vclusterctl/cmd/app/create/pro.go @@ -0,0 +1,178 @@ +package create + +import ( + "context" + "fmt" + "strconv" + "time" + + clusterv1 "github.com/loft-sh/agentapi/v3/pkg/apis/loft/cluster/v1" + managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" + storagev1 "github.com/loft-sh/api/v3/pkg/apis/storage/v1" + "github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create" + proclient "github.com/loft-sh/loftctl/v3/pkg/client" + "github.com/loft-sh/loftctl/v3/pkg/client/helper" + "github.com/loft-sh/loftctl/v3/pkg/client/naming" + "github.com/loft-sh/loftctl/v3/pkg/config" + "github.com/loft-sh/loftctl/v3/pkg/vcluster" + "github.com/loft-sh/log" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func DeployProCluster(ctx context.Context, options *Options, proClient proclient.Client, virtualClusterName string, log log.Logger) error { + // determine project & cluster name + var err error + options.Cluster, options.Project, err = helper.SelectProjectOrCluster(proClient, options.Cluster, options.Project, false, log) + if err != nil { + return err + } + + virtualClusterNamespace := naming.ProjectNamespace(options.Project) + managementClient, err := proClient.Management() + if err != nil { + return err + } + + // make sure there is not existing virtual cluster + var virtualClusterInstance *managementv1.VirtualClusterInstance + virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterNamespace).Get(ctx, virtualClusterName, metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return fmt.Errorf("couldn't retrieve virtual cluster instance: %w", err) + } else if err == nil && !virtualClusterInstance.DeletionTimestamp.IsZero() { + log.Infof("Waiting until virtual cluster is deleted...") + + // wait until the virtual cluster instance is deleted + waitErr := wait.PollUntilContextTimeout(ctx, time.Second, config.Timeout(), false, func(ctx context.Context) (done bool, err error) { + virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterNamespace).Get(ctx, virtualClusterName, metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return false, err + } else if err == nil && virtualClusterInstance.DeletionTimestamp != nil { + return false, nil + } + + return true, nil + }) + if waitErr != nil { + return fmt.Errorf("get virtual cluster instance: %w", err) + } + + virtualClusterInstance = nil + } else if kerrors.IsNotFound(err) { + virtualClusterInstance = nil + } + + // if the virtual cluster already exists and flag is not set, we terminate + if !options.Upgrade && virtualClusterInstance != nil { + return fmt.Errorf("virtual cluster %s already exists in project %s", virtualClusterName, options.Project) + } + + // create virtual cluster if necessary + if virtualClusterInstance == nil { + // resolve template + virtualClusterTemplate, resolvedParameters, err := create.ResolveTemplate( + proClient, + options.Project, + options.Template, + options.TemplateVersion, + options.SetParams, + options.Params, + log, + ) + if err != nil { + return err + } + + // create virtual cluster instance + zone, offset := time.Now().Zone() + virtualClusterInstance = &managementv1.VirtualClusterInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: naming.ProjectNamespace(options.Project), + Name: virtualClusterName, + Annotations: map[string]string{ + clusterv1.SleepModeTimezoneAnnotation: zone + "#" + strconv.Itoa(offset), + }, + }, + Spec: managementv1.VirtualClusterInstanceSpec{ + VirtualClusterInstanceSpec: storagev1.VirtualClusterInstanceSpec{ + TemplateRef: &storagev1.TemplateRef{ + Name: virtualClusterTemplate.Name, + Version: options.TemplateVersion, + }, + ClusterRef: storagev1.VirtualClusterClusterRef{ + ClusterRef: storagev1.ClusterRef{ + Cluster: options.Cluster, + }, + }, + Parameters: resolvedParameters, + }, + }, + } + + // set links + create.SetCustomLinksAnnotation(virtualClusterInstance, options.Links) + + // create virtualclusterinstance + log.Infof("Creating virtual cluster %s in project %s with template %s...", virtualClusterName, options.Project, virtualClusterTemplate.Name) + virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterInstance.Namespace).Create(ctx, virtualClusterInstance, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create virtual cluster: %w", err) + } + } else if options.Upgrade { + // resolve template + virtualClusterTemplate, resolvedParameters, err := create.ResolveTemplate( + proClient, + options.Project, + options.Template, + options.TemplateVersion, + options.SetParams, + options.Params, + log, + ) + if err != nil { + return err + } + + // update virtual cluster instance + if virtualClusterInstance.Spec.TemplateRef == nil { + return fmt.Errorf("virtual cluster instance doesn't use a template, cannot update virtual cluster") + } + + oldVirtualCluster := virtualClusterInstance.DeepCopy() + templateRefChanged := virtualClusterInstance.Spec.TemplateRef.Name != virtualClusterTemplate.Name + paramsChanged := virtualClusterInstance.Spec.Parameters != resolvedParameters + versionChanged := (options.TemplateVersion != "" && virtualClusterInstance.Spec.TemplateRef.Version != options.TemplateVersion) + linksChanged := create.SetCustomLinksAnnotation(virtualClusterInstance, options.Links) + + // check if update is needed + if templateRefChanged || paramsChanged || versionChanged || linksChanged { + virtualClusterInstance.Spec.TemplateRef.Name = virtualClusterTemplate.Name + virtualClusterInstance.Spec.TemplateRef.Version = options.TemplateVersion + virtualClusterInstance.Spec.Parameters = resolvedParameters + + patch := client.MergeFrom(oldVirtualCluster) + patchData, err := patch.Data(virtualClusterInstance) + if err != nil { + return fmt.Errorf("calculate update patch: %w", err) + } + log.Infof("Updating virtual cluster %s in project %s...", virtualClusterName, options.Project) + virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterInstance.Namespace).Patch(ctx, virtualClusterInstance.Name, patch.Type(), patchData, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("patch virtual cluster: %w", err) + } + } else { + log.Infof("Skip updating virtual cluster...") + } + } + + // wait until virtual cluster is ready + virtualClusterInstance, err = vcluster.WaitForVirtualClusterInstance(ctx, managementClient, virtualClusterInstance.Namespace, virtualClusterInstance.Name, true, log) + if err != nil { + return err + } + log.Donef("Successfully created the virtual cluster %s in project %s", virtualClusterName, options.Project) + + return nil +} diff --git a/cmd/vclusterctl/cmd/app/create/types.go b/cmd/vclusterctl/cmd/app/create/types.go index 27b1c25187..42ac4bc67e 100644 --- a/cmd/vclusterctl/cmd/app/create/types.go +++ b/cmd/vclusterctl/cmd/app/create/types.go @@ -7,24 +7,33 @@ type Options struct { ChartName string ChartRepo string LocalChartDir string - K3SImage string Distro string CIDR string - ExtraValues []string + Values []string + SetValues []string + DeprecatedExtraValues []string KubernetesVersion string CreateNamespace bool DisableIngressSync bool - CreateClusterRole bool UpdateCurrent bool Expose bool ExposeLocal bool - Connect bool - Upgrade bool - Isolate bool - ReleaseValues string + Connect bool + Upgrade bool + Isolate bool + + // Pro + Project string + Cluster string + Template string + TemplateVersion string + Links []string + Params string + SetParams []string + DisablePro bool } type Values struct { diff --git a/cmd/vclusterctl/cmd/create.go b/cmd/vclusterctl/cmd/create.go index 5746b4f309..3c1f8098f8 100644 --- a/cmd/vclusterctl/cmd/create.go +++ b/cmd/vclusterctl/cmd/create.go @@ -17,6 +17,7 @@ import ( "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/app/localkubernetes" "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/find" "github.com/loft-sh/vcluster/pkg/embed" + "github.com/loft-sh/vcluster/pkg/pro" "github.com/loft-sh/vcluster/pkg/util/cliconfig" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -88,48 +89,98 @@ vcluster create test --namespace test RunE: func(cobraCmd *cobra.Command, args []string) error { // Check for newer version upgrade.PrintNewerVersionWarning() - validateDeprecated(&cmd.Options, cmd.log) + return cmd.Run(cobraCmd.Context(), args) }, } + // generic flags cobraCmd.Flags().StringVar(&cmd.KubeConfigContextName, "kube-config-context-name", "", "If set, will override the context name of the generated virtual cluster kube config with this name") cobraCmd.Flags().StringVar(&cmd.ChartVersion, "chart-version", upgrade.GetVersion(), "The virtual cluster chart version to use (e.g. v0.9.1)") cobraCmd.Flags().StringVar(&cmd.ChartName, "chart-name", "vcluster", "The virtual cluster chart name to use") cobraCmd.Flags().StringVar(&cmd.ChartRepo, "chart-repo", LoftChartRepo, "The virtual cluster chart repo to use") - cobraCmd.Flags().StringVar(&cmd.LocalChartDir, "local-chart-dir", "", "The virtual cluster local chart dir to use") - cobraCmd.Flags().StringVar(&cmd.K3SImage, "k3s-image", "", "DEPRECATED: use --extra-values instead") cobraCmd.Flags().StringVar(&cmd.Distro, "distro", "k3s", fmt.Sprintf("Kubernetes distro to use for the virtual cluster. Allowed distros: %s", strings.Join(AllowedDistros, ", "))) - cobraCmd.Flags().StringVar(&cmd.ReleaseValues, "release-values", "", "DEPRECATED: use --extra-values instead") cobraCmd.Flags().StringVar(&cmd.KubernetesVersion, "kubernetes-version", "", "The kubernetes version to use (e.g. v1.20). Patch versions are not supported") - cobraCmd.Flags().StringSliceVarP(&cmd.ExtraValues, "extra-values", "f", []string{}, "Path where to load extra helm values from") + cobraCmd.Flags().StringArrayVarP(&cmd.Values, "values", "f", []string{}, "Path where to load extra helm values from") + cobraCmd.Flags().StringArrayVar(&cmd.SetValues, "set", []string{}, "Set values for helm. E.g. --set 'persistence.enabled=true'") + cobraCmd.Flags().StringSliceVar(&cmd.DeprecatedExtraValues, "values", []string{}, "DEPRECATED: use --values instead") + cobraCmd.Flags().BoolVar(&cmd.CreateNamespace, "create-namespace", true, "If true the namespace will be created if it does not exist") - cobraCmd.Flags().BoolVar(&cmd.DisableIngressSync, "disable-ingress-sync", false, "If true the virtual cluster will not sync any ingresses") cobraCmd.Flags().BoolVar(&cmd.UpdateCurrent, "update-current", true, "If true updates the current kube config") - cobraCmd.Flags().BoolVar(&cmd.CreateClusterRole, "create-cluster-role", false, "DEPRECATED: cluster role is now automatically created if it is required by one of the resource syncers that are enabled by the .sync.RESOURCE.enabled=true helm value, which is set in a file that is passed via --extra-values argument.") cobraCmd.Flags().BoolVar(&cmd.Expose, "expose", false, "If true will create a load balancer service to expose the vcluster endpoint") - cobraCmd.Flags().BoolVar(&cmd.ExposeLocal, "expose-local", true, "If true and a local Kubernetes distro is detected, will deploy vcluster with a NodePort service. Will be set to false and the passed value will be ignored if --expose is set to true.") cobraCmd.Flags().BoolVar(&cmd.Connect, "connect", true, "If true will run vcluster connect directly after the vcluster was created") cobraCmd.Flags().BoolVar(&cmd.Upgrade, "upgrade", false, "If true will try to upgrade the vcluster instead of failing if it already exists") cobraCmd.Flags().BoolVar(&cmd.Isolate, "isolate", false, "If true vcluster and its workloads will run in an isolated environment") + cobraCmd.Flags().BoolVar(&cmd.DisablePro, "disable-pro", false, "If true vcluster will not try to create a vCluster.Pro. You can also use 'vcluster logout' to prevent vCluster from creating any pro clusters") + + // pro flags + cobraCmd.Flags().StringVar(&cmd.Project, "project", "", "[PRO] The vCluster.Pro project to use") + cobraCmd.Flags().StringVar(&cmd.Cluster, "cluster", "", "[PRO] The vCluster.Pro connected cluster to use") + cobraCmd.Flags().StringVar(&cmd.Template, "template", "", "[PRO] The vCluster.Pro template to use") + cobraCmd.Flags().StringVar(&cmd.TemplateVersion, "template-version", "", "[PRO] The vCluster.Pro template version to use") + cobraCmd.Flags().StringArrayVar(&cmd.Links, "link", []string{}, "[PRO] A link to add to the vCluster. E.g. --link 'prod=http://exampleprod.com'") + cobraCmd.Flags().StringVar(&cmd.Params, "params", "", "[PRO] If a template is used, this can be used to use a file for the parameters. E.g. --params path/to/my/file.yaml") + cobraCmd.Flags().StringArrayVar(&cmd.SetParams, "set-param", []string{}, "[PRO] If a template is used, this can be used to set a specific parameter. E.g. --set-param 'my-param=my-value'") + + // hidden / deprecated + cobraCmd.Flags().StringVar(&cmd.LocalChartDir, "local-chart-dir", "", "The virtual cluster local chart dir to use") + cobraCmd.Flags().BoolVar(&cmd.DisableIngressSync, "disable-ingress-sync", false, "DEPRECATED: use --set 'sync.ingresses.enabled=false'") + cobraCmd.Flags().BoolVar(&cmd.ExposeLocal, "expose-local", true, "If true and a local Kubernetes distro is detected, will deploy vcluster with a NodePort service. Will be set to false and the passed value will be ignored if --expose is set to true.") + + _ = cobraCmd.Flags().MarkHidden("local-chart-dir") + _ = cobraCmd.Flags().MarkHidden("disable-ingress-sync") + _ = cobraCmd.Flags().MarkHidden("expose-local") return cobraCmd } -func validateDeprecated(createOptions *create.Options, log log.Logger) { - if createOptions.ReleaseValues != "" { - log.Warn("Flag --release-values is deprecated, please use --extra-values instead. This flag will be removed in future!") +var loginText = "\nPlease run:\n * 'vcluster login' to connect to an existing vCluster.Pro instance\n * 'vcluster pro start' to deploy a new vCluster.Pro instance" + +func (cmd *CreateCmd) validateOSSFlags() error { + if cmd.Project != "" { + return fmt.Errorf("cannot use --project as you are not connected to a vCluster.Pro instance." + loginText) + } + if cmd.Cluster != "" { + return fmt.Errorf("cannot use --cluster as you are not connected to a vCluster.Pro instance." + loginText) + } + if cmd.Template != "" { + return fmt.Errorf("cannot use --template as you are not connected to a vCluster.Pro instance." + loginText) } - if createOptions.K3SImage != "" { - log.Warn("Flag --k3s-image is deprecated, please use --extra-values instead. This flag will be removed in future!") + if cmd.TemplateVersion != "" { + return fmt.Errorf("cannot use --template-version as you are not connected to a vCluster.Pro instance." + loginText) } - if createOptions.CreateClusterRole { - log.Warn("Flag --create-cluster-role is deprecated. Cluster role is now automatically created if it is required by one of the resource syncers that are enabled by the .sync.RESOURCE.enabled=true helm value, which is set in a file that is passed via --extra-values (or -f) argument.") + if len(cmd.Links) > 0 { + return fmt.Errorf("cannot use --link as you are not connected to a vCluster.Pro instance." + loginText) } + if cmd.Params != "" { + return fmt.Errorf("cannot use --params as you are not connected to a vCluster.Pro instance." + loginText) + } + if len(cmd.SetParams) > 0 { + return fmt.Errorf("cannot use --set-params as you are not connected to a vCluster.Pro instance." + loginText) + } + + return nil } // Run executes the functionality func (cmd *CreateCmd) Run(ctx context.Context, args []string) error { + cmd.Values = append(cmd.Values, cmd.DeprecatedExtraValues...) + + // check if we should create a pro cluster + if !cmd.DisablePro { + proClient, err := pro.CreateProClient() + if err == nil { + return create.DeployProCluster(ctx, &cmd.Options, proClient, args[0], cmd.log) + } + } + + // validate oss flags + err := cmd.validateOSSFlags() + if err != nil { + return err + } + + // check helm binary helmBinaryPath, err := GetHelmBinaryPath(ctx, cmd.log) if err != nil { return err @@ -138,8 +189,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error { output, err := exec.Command(helmBinaryPath, "version", "--client").CombinedOutput() if errHelm := CheckHelmVersion(string(output)); errHelm != nil { return errHelm - } - if err != nil { + } else if err != nil { return err } @@ -166,7 +216,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error { } var newExtraValues []string - for _, value := range cmd.ExtraValues { + for _, value := range cmd.Values { decodedString, err := getBase64DecodedString(value) // ignore decoding errors and treat it as non-base64 string if err != nil { @@ -198,11 +248,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error { } // resetting this as the base64 encoded strings should be removed and only valid file names should be kept. - cmd.ExtraValues = newExtraValues - - if cmd.ReleaseValues != "" { - cmd.ExtraValues = append(cmd.ExtraValues, cmd.ReleaseValues) - } + cmd.Values = newExtraValues // check if vcluster already exists if !cmd.Upgrade { @@ -325,7 +371,8 @@ func (cmd *CreateCmd) deployChart(ctx context.Context, vClusterName, chartValues Repo: cmd.ChartRepo, Version: cmd.ChartVersion, Values: chartValues, - ValuesFiles: cmd.ExtraValues, + ValuesFiles: cmd.Values, + SetValues: cmd.SetValues, }) if err != nil { return err @@ -371,12 +418,11 @@ func (cmd *CreateCmd) ToChartOptions(kubernetesVersion *version.Info) (*helmUtil ChartRepo: cmd.ChartRepo, ChartVersion: cmd.ChartVersion, CIDR: cmd.CIDR, - CreateClusterRole: cmd.CreateClusterRole, + CreateClusterRole: true, DisableIngressSync: cmd.DisableIngressSync, Expose: cmd.Expose, SyncNodes: cmd.localCluster, NodePort: cmd.localCluster, - K3SImage: cmd.K3SImage, Isolate: cmd.Isolate, KubernetesVersion: helmUtils.Version{ Major: kubernetesVersion.Major, diff --git a/cmd/vclusterctl/cmd/delete.go b/cmd/vclusterctl/cmd/delete.go index cecb719ed8..2afede36b7 100644 --- a/cmd/vclusterctl/cmd/delete.go +++ b/cmd/vclusterctl/cmd/delete.go @@ -4,12 +4,16 @@ import ( "context" "fmt" "os/exec" + "time" + proclient "github.com/loft-sh/loftctl/v3/pkg/client" + "github.com/loft-sh/loftctl/v3/pkg/client/naming" + "github.com/loft-sh/loftctl/v3/pkg/kube" loftctlUtil "github.com/loft-sh/loftctl/v3/pkg/util" - "github.com/loft-sh/vcluster/pkg/util/translate" - "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/app/localkubernetes" "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/find" + "github.com/loft-sh/vcluster/pkg/pro" + "github.com/loft-sh/vcluster/pkg/util/translate" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -29,6 +33,8 @@ import ( type DeleteCmd struct { *flags.GlobalFlags + Project string + Wait bool KeepPVC bool DeleteNamespace bool AutoDeleteNamespace bool @@ -67,6 +73,8 @@ vcluster delete test --namespace test }, } + cobraCmd.Flags().StringVar(&cmd.Project, "project", "", "[PRO] The pro project the vcluster is in") + cobraCmd.Flags().BoolVar(&cmd.Wait, "wait", true, "If enabled, vcluster will wait until the vcluster is deleted") cobraCmd.Flags().BoolVar(&cmd.KeepPVC, "keep-pvc", false, "If enabled, vcluster will not delete the persistent volume claim of the vcluster") cobraCmd.Flags().BoolVar(&cmd.DeleteNamespace, "delete-namespace", false, "If enabled, vcluster will delete the namespace of the vcluster. In the case of multi-namespace mode, will also delete all other namespaces created by vcluster") cobraCmd.Flags().BoolVar(&cmd.AutoDeleteNamespace, "auto-delete-namespace", true, "If enabled, vcluster will delete the namespace of the vcluster if it was created by vclusterctl. In the case of multi-namespace mode, will also delete all other namespaces created by vcluster") @@ -77,6 +85,27 @@ vcluster delete test --namespace test func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { ctx := cobraCmd.Context() + // get pro client + proClient, err := pro.CreateProClient() + if err != nil { + cmd.log.Debugf("Error creating pro client: %v", err) + } + + // find vcluster + vClusterName := args[0] + vCluster, proVCluster, err := find.GetVCluster(ctx, proClient, cmd.Context, vClusterName, cmd.Namespace, cmd.Project, cmd.log) + if err != nil { + return err + } else if proVCluster != nil { + return cmd.deleteProVCluster(cobraCmd.Context(), proClient, proVCluster) + } + + // prepare client + err = cmd.prepare(vCluster) + if err != nil { + return err + } + // test for helm helmBinaryPath, err := GetHelmBinaryPath(ctx, cmd.log) if err != nil { @@ -91,12 +120,6 @@ func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { return fmt.Errorf("seems like there are issues with your helm client: \n\n%s", output) } - // prepare client - err = cmd.prepare(cobraCmd.Context(), args[0]) - if err != nil { - return err - } - // check if namespace if cmd.AutoDeleteNamespace { namespace, err := cmd.kubeClient.CoreV1().Namespaces().Get(ctx, cmd.Namespace, metav1.GetOptions{}) @@ -162,6 +185,7 @@ func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { return err } + // delete namespace err = client.CoreV1().Namespaces().Delete(ctx, cmd.Namespace, metav1.DeleteOptions{}) if err != nil { if !kerrors.IsNotFound(err) { @@ -192,17 +216,64 @@ func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { } } } + + // wait for vcluster deletion + if cmd.Wait { + cmd.log.Info("Waiting for virtual cluster to be deleted...") + for { + _, err = client.CoreV1().Namespaces().Get(ctx, cmd.Namespace, metav1.GetOptions{}) + if err != nil { + break + } + + time.Sleep(time.Second) + } + cmd.log.Done("Virtual Cluster is deleted") + } } return nil } -func (cmd *DeleteCmd) prepare(ctx context.Context, vClusterName string) error { - vCluster, _, err := find.GetVCluster(ctx, nil, cmd.Context, vClusterName, cmd.Namespace, "", cmd.log) +func (cmd *DeleteCmd) deleteProVCluster(ctx context.Context, proClient proclient.Client, vCluster *pro.VirtualClusterInstanceProject) error { + managementClient, err := proClient.Management() if err != nil { return err } + cmd.log.Infof("Deleting virtual cluster %s in project %s", vCluster.VirtualCluster.Name, vCluster.Project.Name) + + err = managementClient.Loft().ManagementV1().VirtualClusterInstances(naming.ProjectNamespace(vCluster.VirtualCluster.Namespace)).Delete(ctx, vCluster.VirtualCluster.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrap(err, "delete virtual cluster") + } + + cmd.log.Donef("Successfully deleted virtual cluster %s in project %s", vCluster.VirtualCluster.Name, vCluster.Project.Name) + + // update kube config + err = deleteProContext(vCluster.VirtualCluster.Name, vCluster.VirtualCluster.Namespace) + if err != nil { + return errors.Wrap(err, "delete kube context") + } + + // wait until deleted + if cmd.Wait { + cmd.log.Info("Waiting for virtual cluster to be deleted...") + for isVirtualClusterInstanceStillThere(ctx, managementClient, vCluster.VirtualCluster.Namespace, vCluster.VirtualCluster.Name) { + time.Sleep(time.Second) + } + cmd.log.Done("Virtual Cluster is deleted") + } + + return nil +} + +func isVirtualClusterInstanceStillThere(ctx context.Context, managementClient kube.Interface, namespace, name string) bool { + _, err := managementClient.Loft().ManagementV1().VirtualClusterInstances(namespace).Get(ctx, name, metav1.GetOptions{}) + return err == nil +} + +func (cmd *DeleteCmd) prepare(vCluster *find.VCluster) error { // load the raw config rawConfig, err := vCluster.ClientFactory.RawConfig() if err != nil { @@ -219,13 +290,13 @@ func (cmd *DeleteCmd) prepare(ctx context.Context, vClusterName string) error { return err } - err = localkubernetes.CleanupLocal(vClusterName, vCluster.Namespace, &rawConfig, cmd.log) + err = localkubernetes.CleanupLocal(vCluster.Name, vCluster.Namespace, &rawConfig, cmd.log) if err != nil { cmd.log.Warnf("error cleaning up: %v", err) } // construct proxy name - proxyName := find.VClusterConnectBackgroundProxyName(vClusterName, vCluster.Namespace, rawConfig.CurrentContext) + proxyName := find.VClusterConnectBackgroundProxyName(vCluster.Name, vCluster.Namespace, rawConfig.CurrentContext) _ = localkubernetes.CleanupBackgroundProxy(proxyName, cmd.log) kubeClient, err := kubernetes.NewForConfig(restConfig) @@ -240,6 +311,29 @@ func (cmd *DeleteCmd) prepare(ctx context.Context, vClusterName string) error { return nil } +func deleteProContext(vClusterName, vClusterNamespace string) error { + kubeClientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + kubeConfig, err := kubeClientConfig.RawConfig() + if err != nil { + return fmt.Errorf("load kube config: %w", err) + } + + // remove matching contexts + for contextName := range kubeConfig.Contexts { + name, namespace, previousContext := find.VClusterProFromContext(contextName) + if vClusterName != name || vClusterNamespace != namespace { + continue + } + + err := deleteContext(&kubeConfig, contextName, previousContext) + if err != nil { + return err + } + } + + return nil +} + func deleteContext(kubeConfig *clientcmdapi.Config, kubeContext string, otherContext string) error { // Get context contextRaw, ok := kubeConfig.Contexts[kubeContext] diff --git a/cmd/vclusterctl/cmd/find/find.go b/cmd/vclusterctl/cmd/find/find.go index d9ba9cbbfa..42e02c2fd4 100644 --- a/cmd/vclusterctl/cmd/find/find.go +++ b/cmd/vclusterctl/cmd/find/find.go @@ -145,6 +145,13 @@ func formatOptions(format string, options [][]string) []string { func ListVClusters(ctx context.Context, proClient proclient.Client, context, name, namespace, project string, log log.Logger) ([]VCluster, []pro.VirtualClusterInstanceProject, error) { var err error + if context == "" { + var err error + context, _, err = CurrentContext() + if err != nil { + return nil, nil, err + } + } var ossVClusters []VCluster if project == "" { @@ -166,13 +173,7 @@ func ListVClusters(ctx context.Context, proClient proclient.Client, context, nam } func listOSSVClusters(ctx context.Context, context, name, namespace string) ([]VCluster, error) { - if context == "" { - var err error - context, _, err = CurrentContext() - if err != nil { - return nil, err - } - } + var err error vClusterName, _, vClusterContext := VClusterFromContext(context) timeout := time.Minute @@ -180,24 +181,32 @@ func listOSSVClusters(ctx context.Context, context, name, namespace string) ([]V timeout = time.Second * 5 } - // In case of error in vcluster listing in vcluster context, the below check will skip the error and try searching for parent context vclusters. - vclusters, err := findInContext(ctx, context, name, namespace, timeout, false) - if err != nil && vClusterName == "" { - return nil, errors.Wrap(err, "find vcluster") + vclusters := []VCluster{} + isPro := strings.HasPrefix(context, "vcluster-pro_") + if !isPro { + // In case of error in vcluster listing in vcluster context, the below check will skip the error and try searching for parent context vclusters. + vclusters, err = findInContext(ctx, context, name, namespace, timeout, false) + if err != nil && vClusterName == "" { + return nil, errors.Wrap(err, "find vcluster") + } } if vClusterName != "" { - parentContextVclusters, err := findInContext(ctx, vClusterContext, name, namespace, time.Minute, true) + parentContextVClusters, err := findInContext(ctx, vClusterContext, name, namespace, time.Minute, true) if err != nil { return nil, errors.Wrap(err, "find vcluster") } - vclusters = append(vclusters, parentContextVclusters...) + vclusters = append(vclusters, parentContextVClusters...) } return vclusters, nil } +func VClusterProContextName(vClusterName string, vClusterNamespace string, currentContext string) string { + return "vcluster-pro_" + vClusterName + "_" + vClusterNamespace + "_" + currentContext +} + func VClusterContextName(vClusterName string, vClusterNamespace string, currentContext string) string { return "vcluster_" + vClusterName + "_" + vClusterNamespace + "_" + currentContext } @@ -206,6 +215,21 @@ func VClusterConnectBackgroundProxyName(vClusterName string, vClusterNamespace s return VClusterContextName(vClusterName, vClusterNamespace, currentContext) + "_background_proxy" } +func VClusterProFromContext(originalContext string) (name string, namespace string, context string) { + if !strings.HasPrefix(originalContext, "vcluster-pro_") { + return "", "", "" + } + + splitted := strings.Split(originalContext, "_") + // vcluster-pro___ + if len(splitted) >= 4 { + return splitted[1], splitted[2], strings.Join(splitted[3:], "_") + } + + // we don't know for sure, but most likely specified custom vcluster context name + return originalContext, "", "" +} + func VClusterFromContext(originalContext string) (name string, namespace string, context string) { if !strings.HasPrefix(originalContext, "vcluster_") { return "", "", "" diff --git a/go.mod b/go.mod index ddaacad0f4..3e5f63d6eb 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/loft-sh/api/v3 v3.0.0-20230922094800-6d0c1cbf0fa6 - github.com/loft-sh/loftctl/v3 v3.0.0-20230922094952-2a6aef29f31e + github.com/loft-sh/loftctl/v3 v3.0.0-20230922100229-83e8d117af6d github.com/loft-sh/utils v0.0.25 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/term v0.5.0 diff --git a/go.sum b/go.sum index c31876880f..e0ba35d120 100644 --- a/go.sum +++ b/go.sum @@ -564,6 +564,8 @@ github.com/loft-sh/loftctl/v3 v3.0.0-20230921143437-669b265e3ecf h1:SwIal5P1coM0 github.com/loft-sh/loftctl/v3 v3.0.0-20230921143437-669b265e3ecf/go.mod h1:I+dMG4zKZWOn+CcteN0cyPGOclHlqqmXBC6THcUwj60= github.com/loft-sh/loftctl/v3 v3.0.0-20230922094952-2a6aef29f31e h1:OhQRp9uH7U14lYg8yOrnOiVhhxxC9tFdwWSyxoSUJ/I= github.com/loft-sh/loftctl/v3 v3.0.0-20230922094952-2a6aef29f31e/go.mod h1:g2HZHmpkqtiQGqM8QVRuaCgNDI6IMW6DZQP0cOUgxp8= +github.com/loft-sh/loftctl/v3 v3.0.0-20230922100229-83e8d117af6d h1:bmOCghp4sNtzw0V8xwhMt5myeL+w2qzO3zHeRrwDpRE= +github.com/loft-sh/loftctl/v3 v3.0.0-20230922100229-83e8d117af6d/go.mod h1:g2HZHmpkqtiQGqM8QVRuaCgNDI6IMW6DZQP0cOUgxp8= github.com/loft-sh/log v0.0.0-20230824104949-bd516c25712a h1:/gqqjKpcHEdFXIX41lx1Y/FBqT/72gbPpf7sa20tyM8= github.com/loft-sh/log v0.0.0-20230824104949-bd516c25712a/go.mod h1:YImeRjXH34Yf5E79T7UHBQpDZl9fIaaFRgyZ/bkY+UQ= github.com/loft-sh/utils v0.0.25 h1:JbbRJfXO1Rd34fQcaoDSmwyPBEzsrLwBSR21C90hHuk= diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 752503da56..a48374ae46 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -24,8 +24,8 @@ type UpgradeOptions struct { Version string Values string ValuesFiles []string - SetValues map[string]string - SetStringValues map[string]string + SetValues []string + SetStringValues []string CreateNamespace bool @@ -140,37 +140,16 @@ func (c *client) run(ctx context.Context, name, namespace string, options Upgrad } // Set values - if options.SetValues != nil && len(options.SetValues) > 0 { - args = append(args, "--set") - - setString := "" - for key, value := range options.SetValues { - if setString != "" { - setString += "," - } - - setString += key + "=" + value - } - - args = append(args, setString) + for _, value := range options.SetValues { + args = append(args, "--set", value) } // Set string values - if options.SetStringValues != nil && len(options.SetStringValues) > 0 { - args = append(args, "--set-string") - - setString := "" - for key, value := range options.SetStringValues { - if setString != "" { - setString += "," - } - - setString += key + "=" + value - } - - args = append(args, setString) + for _, value := range options.SetStringValues { + args = append(args, "--set-string", value) } + // force if options.Force { args = append(args, "--force") } diff --git a/pkg/pro/create.go b/pkg/pro/create.go new file mode 100644 index 0000000000..d27a828d4a --- /dev/null +++ b/pkg/pro/create.go @@ -0,0 +1 @@ +package pro diff --git a/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/space.go b/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/space.go index 35b5f2950b..4cda199f83 100644 --- a/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/space.go +++ b/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/space.go @@ -154,7 +154,7 @@ func (cmd *SpaceCmd) Run(ctx context.Context, args []string) error { } // determine cluster name - cmd.Cluster, cmd.Project, err = helper.SelectProjectOrCluster(baseClient, cmd.Cluster, cmd.Project, cmd.Log) + cmd.Cluster, cmd.Project, err = helper.SelectProjectOrCluster(baseClient, cmd.Cluster, cmd.Project, true, cmd.Log) if err != nil { return err } diff --git a/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/vcluster.go b/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/vcluster.go index d58ba888f8..612355d89e 100644 --- a/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/vcluster.go +++ b/vendor/github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create/vcluster.go @@ -168,7 +168,7 @@ func (cmd *VirtualClusterCmd) Run(ctx context.Context, args []string) error { } // determine cluster name - cmd.Cluster, cmd.Project, err = helper.SelectProjectOrCluster(baseClient, cmd.Cluster, cmd.Project, cmd.Log) + cmd.Cluster, cmd.Project, err = helper.SelectProjectOrCluster(baseClient, cmd.Cluster, cmd.Project, true, cmd.Log) if err != nil { return err } @@ -217,9 +217,8 @@ func (cmd *VirtualClusterCmd) createVirtualCluster(ctx context.Context, baseClie } } - var virtualClusterInstance *managementv1.VirtualClusterInstance - // make sure there is not existing virtual cluster + var virtualClusterInstance *managementv1.VirtualClusterInstance virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterNamespace).Get(ctx, virtualClusterName, metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { return fmt.Errorf("couldn't retrieve virtual cluster instance: %w", err) @@ -254,7 +253,15 @@ func (cmd *VirtualClusterCmd) createVirtualCluster(ctx context.Context, baseClie // create virtual cluster if necessary if virtualClusterInstance == nil { // resolve template - virtualClusterTemplate, resolvedParameters, err := cmd.resolveTemplate(baseClient) + virtualClusterTemplate, resolvedParameters, err := ResolveTemplate( + baseClient, + cmd.Project, + cmd.Template, + cmd.Version, + cmd.Set, + cmd.ParametersFile, + cmd.Log, + ) if err != nil { return err } @@ -282,13 +289,18 @@ func (cmd *VirtualClusterCmd) createVirtualCluster(ctx context.Context, baseClie Version: cmd.Version, }, ClusterRef: storagev1.VirtualClusterClusterRef{ - ClusterRef: storagev1.ClusterRef{Cluster: cmd.Cluster}, + ClusterRef: storagev1.ClusterRef{ + Cluster: cmd.Cluster, + }, }, Parameters: resolvedParameters, }, }, } + + // set links SetCustomLinksAnnotation(virtualClusterInstance, cmd.Links) + // create virtualclusterinstance cmd.Log.Infof("Creating virtual cluster %s in project %s with template %s...", ansi.Color(virtualClusterName, "white+b"), ansi.Color(cmd.Project, "white+b"), ansi.Color(virtualClusterTemplate.Name, "white+b")) virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterInstance.Namespace).Create(ctx, virtualClusterInstance, metav1.CreateOptions{}) @@ -297,7 +309,15 @@ func (cmd *VirtualClusterCmd) createVirtualCluster(ctx context.Context, baseClie } } else if cmd.Update { // resolve template - virtualClusterTemplate, resolvedParameters, err := cmd.resolveTemplate(baseClient) + virtualClusterTemplate, resolvedParameters, err := ResolveTemplate( + baseClient, + cmd.Project, + cmd.Template, + cmd.Version, + cmd.Set, + cmd.ParametersFile, + cmd.Log, + ) if err != nil { return err } @@ -361,9 +381,17 @@ func (cmd *VirtualClusterCmd) createVirtualCluster(ctx context.Context, baseClie return nil } -func (cmd *VirtualClusterCmd) resolveTemplate(baseClient client.Client) (*managementv1.VirtualClusterTemplate, string, error) { +func ResolveTemplate( + baseClient client.Client, + project, + template, + templateVersion string, + setParams []string, + fileParams string, + log log.Logger, +) (*managementv1.VirtualClusterTemplate, string, error) { // determine space template to use - virtualClusterTemplate, err := helper.SelectVirtualClusterTemplate(baseClient, cmd.Project, cmd.Template, cmd.Log) + virtualClusterTemplate, err := helper.SelectVirtualClusterTemplate(baseClient, project, template, log) if err != nil { return nil, "", err } @@ -371,7 +399,7 @@ func (cmd *VirtualClusterCmd) resolveTemplate(baseClient client.Client) (*manage // get parameters var templateParameters []storagev1.AppParameter if len(virtualClusterTemplate.Spec.Versions) > 0 { - if cmd.Version == "" { + if templateVersion == "" { latestVersion := version.GetLatestVersion(virtualClusterTemplate) if latestVersion == nil { return nil, "", fmt.Errorf("couldn't find any version in template") @@ -379,11 +407,11 @@ func (cmd *VirtualClusterCmd) resolveTemplate(baseClient client.Client) (*manage templateParameters = latestVersion.(*storagev1.VirtualClusterTemplateVersion).Parameters } else { - _, latestMatched, err := version.GetLatestMatchedVersion(virtualClusterTemplate, cmd.Version) + _, latestMatched, err := version.GetLatestMatchedVersion(virtualClusterTemplate, templateVersion) if err != nil { return nil, "", err } else if latestMatched == nil { - return nil, "", fmt.Errorf("couldn't find any matching version to %s", cmd.Version) + return nil, "", fmt.Errorf("couldn't find any matching version to %s", templateVersion) } templateParameters = latestMatched.(*storagev1.VirtualClusterTemplateVersion).Parameters @@ -393,7 +421,7 @@ func (cmd *VirtualClusterCmd) resolveTemplate(baseClient client.Client) (*manage } // resolve space template parameters - resolvedParameters, err := parameters.ResolveTemplateParameters(cmd.Set, templateParameters, cmd.ParametersFile) + resolvedParameters, err := parameters.ResolveTemplateParameters(setParams, templateParameters, fileParams) if err != nil { return nil, "", err } diff --git a/vendor/github.com/loft-sh/loftctl/v3/pkg/client/helper/helper.go b/vendor/github.com/loft-sh/loftctl/v3/pkg/client/helper/helper.go index fb53ebfe83..5b98e5bd08 100644 --- a/vendor/github.com/loft-sh/loftctl/v3/pkg/client/helper/helper.go +++ b/vendor/github.com/loft-sh/loftctl/v3/pkg/client/helper/helper.go @@ -363,10 +363,10 @@ func SelectSpaceInstanceOrSpace(baseClient client.Client, spaceName, projectName return "", "", "", fmt.Errorf("couldn't find answer") } -func SelectProjectOrCluster(baseClient client.Client, clusterName, projectName string, log log.Logger) (cluster string, project string, err error) { +func SelectProjectOrCluster(baseClient client.Client, clusterName, projectName string, allowClusterOnly bool, log log.Logger) (cluster string, project string, err error) { if projectName != "" { return clusterName, projectName, nil - } else if clusterName != "" { + } else if allowClusterOnly && clusterName != "" { return clusterName, "", nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index d03b8f4d1a..d6dfc65a7e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -340,7 +340,7 @@ github.com/loft-sh/external-types/loft-sh/admin-services/pkg/server # github.com/loft-sh/jspolicy v0.1.0 ## explicit; go 1.16 github.com/loft-sh/jspolicy/pkg/apis/policy/v1beta1 -# github.com/loft-sh/loftctl/v3 v3.0.0-20230922094952-2a6aef29f31e +# github.com/loft-sh/loftctl/v3 v3.0.0-20230922100229-83e8d117af6d ## explicit; go 1.21.1 github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/connect