Skip to content

Commit

Permalink
feat: allow vcluster pro via vcluster create & delete
Browse files Browse the repository at this point in the history
  • Loading branch information
FabianKramm committed Sep 22, 2023
1 parent a54e5cf commit 27ef31b
Show file tree
Hide file tree
Showing 13 changed files with 464 additions and 103 deletions.
178 changes: 178 additions & 0 deletions cmd/vclusterctl/cmd/app/create/pro.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
package create

import (
"context"
"fmt"
"strconv"
"time"

clusterv1 "github.com/loft-sh/agentapi/v3/pkg/apis/loft/cluster/v1"
managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1"
storagev1 "github.com/loft-sh/api/v3/pkg/apis/storage/v1"
"github.com/loft-sh/loftctl/v3/cmd/loftctl/cmd/create"
proclient "github.com/loft-sh/loftctl/v3/pkg/client"
"github.com/loft-sh/loftctl/v3/pkg/client/helper"
"github.com/loft-sh/loftctl/v3/pkg/client/naming"
"github.com/loft-sh/loftctl/v3/pkg/config"
"github.com/loft-sh/loftctl/v3/pkg/vcluster"
"github.com/loft-sh/log"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func DeployProCluster(ctx context.Context, options *Options, proClient proclient.Client, virtualClusterName string, log log.Logger) error {
// determine project & cluster name
var err error
options.Cluster, options.Project, err = helper.SelectProjectOrCluster(proClient, options.Cluster, options.Project, false, log)
if err != nil {
return err
}

virtualClusterNamespace := naming.ProjectNamespace(options.Project)
managementClient, err := proClient.Management()
if err != nil {
return err
}

// make sure there is not existing virtual cluster
var virtualClusterInstance *managementv1.VirtualClusterInstance
virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterNamespace).Get(ctx, virtualClusterName, metav1.GetOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return fmt.Errorf("couldn't retrieve virtual cluster instance: %w", err)
} else if err == nil && !virtualClusterInstance.DeletionTimestamp.IsZero() {
log.Infof("Waiting until virtual cluster is deleted...")

// wait until the virtual cluster instance is deleted
waitErr := wait.PollUntilContextTimeout(ctx, time.Second, config.Timeout(), false, func(ctx context.Context) (done bool, err error) {
virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterNamespace).Get(ctx, virtualClusterName, metav1.GetOptions{})
if err != nil && !kerrors.IsNotFound(err) {
return false, err
} else if err == nil && virtualClusterInstance.DeletionTimestamp != nil {
return false, nil
}

return true, nil
})
if waitErr != nil {
return fmt.Errorf("get virtual cluster instance: %w", err)
}

virtualClusterInstance = nil
} else if kerrors.IsNotFound(err) {
virtualClusterInstance = nil
}

// if the virtual cluster already exists and flag is not set, we terminate
if !options.Upgrade && virtualClusterInstance != nil {
return fmt.Errorf("virtual cluster %s already exists in project %s", virtualClusterName, options.Project)
}

// create virtual cluster if necessary
if virtualClusterInstance == nil {
// resolve template
virtualClusterTemplate, resolvedParameters, err := create.ResolveTemplate(
proClient,
options.Project,
options.Template,
options.TemplateVersion,
options.SetParams,
options.Params,
log,
)
if err != nil {
return err
}

// create virtual cluster instance
zone, offset := time.Now().Zone()
virtualClusterInstance = &managementv1.VirtualClusterInstance{
ObjectMeta: metav1.ObjectMeta{
Namespace: naming.ProjectNamespace(options.Project),
Name: virtualClusterName,
Annotations: map[string]string{
clusterv1.SleepModeTimezoneAnnotation: zone + "#" + strconv.Itoa(offset),
},
},
Spec: managementv1.VirtualClusterInstanceSpec{
VirtualClusterInstanceSpec: storagev1.VirtualClusterInstanceSpec{
TemplateRef: &storagev1.TemplateRef{
Name: virtualClusterTemplate.Name,
Version: options.TemplateVersion,
},
ClusterRef: storagev1.VirtualClusterClusterRef{
ClusterRef: storagev1.ClusterRef{
Cluster: options.Cluster,
},
},
Parameters: resolvedParameters,
},
},
}

// set links
create.SetCustomLinksAnnotation(virtualClusterInstance, options.Links)

// create virtualclusterinstance
log.Infof("Creating virtual cluster %s in project %s with template %s...", virtualClusterName, options.Project, virtualClusterTemplate.Name)
virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterInstance.Namespace).Create(ctx, virtualClusterInstance, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("create virtual cluster: %w", err)
}
} else if options.Upgrade {
// resolve template
virtualClusterTemplate, resolvedParameters, err := create.ResolveTemplate(
proClient,
options.Project,
options.Template,
options.TemplateVersion,
options.SetParams,
options.Params,
log,
)
if err != nil {
return err
}

// update virtual cluster instance
if virtualClusterInstance.Spec.TemplateRef == nil {
return fmt.Errorf("virtual cluster instance doesn't use a template, cannot update virtual cluster")
}

oldVirtualCluster := virtualClusterInstance.DeepCopy()
templateRefChanged := virtualClusterInstance.Spec.TemplateRef.Name != virtualClusterTemplate.Name
paramsChanged := virtualClusterInstance.Spec.Parameters != resolvedParameters
versionChanged := (options.TemplateVersion != "" && virtualClusterInstance.Spec.TemplateRef.Version != options.TemplateVersion)
linksChanged := create.SetCustomLinksAnnotation(virtualClusterInstance, options.Links)

// check if update is needed
if templateRefChanged || paramsChanged || versionChanged || linksChanged {
virtualClusterInstance.Spec.TemplateRef.Name = virtualClusterTemplate.Name
virtualClusterInstance.Spec.TemplateRef.Version = options.TemplateVersion
virtualClusterInstance.Spec.Parameters = resolvedParameters

patch := client.MergeFrom(oldVirtualCluster)
patchData, err := patch.Data(virtualClusterInstance)
if err != nil {
return fmt.Errorf("calculate update patch: %w", err)
}
log.Infof("Updating virtual cluster %s in project %s...", virtualClusterName, options.Project)
virtualClusterInstance, err = managementClient.Loft().ManagementV1().VirtualClusterInstances(virtualClusterInstance.Namespace).Patch(ctx, virtualClusterInstance.Name, patch.Type(), patchData, metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("patch virtual cluster: %w", err)
}
} else {
log.Infof("Skip updating virtual cluster...")
}
}

// wait until virtual cluster is ready
virtualClusterInstance, err = vcluster.WaitForVirtualClusterInstance(ctx, managementClient, virtualClusterInstance.Namespace, virtualClusterInstance.Name, true, log)
if err != nil {
return err
}
log.Donef("Successfully created the virtual cluster %s in project %s", virtualClusterName, options.Project)

return nil
}
23 changes: 16 additions & 7 deletions cmd/vclusterctl/cmd/app/create/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,33 @@ type Options struct {
ChartName string
ChartRepo string
LocalChartDir string
K3SImage string
Distro string
CIDR string
ExtraValues []string
Values []string
SetValues []string
DeprecatedExtraValues []string

KubernetesVersion string

CreateNamespace bool
DisableIngressSync bool
CreateClusterRole bool
UpdateCurrent bool
Expose bool
ExposeLocal bool

Connect bool
Upgrade bool
Isolate bool
ReleaseValues string
Connect bool
Upgrade bool
Isolate bool

// Pro
Project string
Cluster string
Template string
TemplateVersion string
Links []string
Params string
SetParams []string
DisablePro bool
}

type Values struct {
Expand Down
98 changes: 72 additions & 26 deletions cmd/vclusterctl/cmd/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/app/localkubernetes"
"github.com/loft-sh/vcluster/cmd/vclusterctl/cmd/find"
"github.com/loft-sh/vcluster/pkg/embed"
"github.com/loft-sh/vcluster/pkg/pro"
"github.com/loft-sh/vcluster/pkg/util/cliconfig"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -88,48 +89,98 @@ vcluster create test --namespace test
RunE: func(cobraCmd *cobra.Command, args []string) error {
// Check for newer version
upgrade.PrintNewerVersionWarning()
validateDeprecated(&cmd.Options, cmd.log)

return cmd.Run(cobraCmd.Context(), args)
},
}

// generic flags
cobraCmd.Flags().StringVar(&cmd.KubeConfigContextName, "kube-config-context-name", "", "If set, will override the context name of the generated virtual cluster kube config with this name")
cobraCmd.Flags().StringVar(&cmd.ChartVersion, "chart-version", upgrade.GetVersion(), "The virtual cluster chart version to use (e.g. v0.9.1)")
cobraCmd.Flags().StringVar(&cmd.ChartName, "chart-name", "vcluster", "The virtual cluster chart name to use")
cobraCmd.Flags().StringVar(&cmd.ChartRepo, "chart-repo", LoftChartRepo, "The virtual cluster chart repo to use")
cobraCmd.Flags().StringVar(&cmd.LocalChartDir, "local-chart-dir", "", "The virtual cluster local chart dir to use")
cobraCmd.Flags().StringVar(&cmd.K3SImage, "k3s-image", "", "DEPRECATED: use --extra-values instead")
cobraCmd.Flags().StringVar(&cmd.Distro, "distro", "k3s", fmt.Sprintf("Kubernetes distro to use for the virtual cluster. Allowed distros: %s", strings.Join(AllowedDistros, ", ")))
cobraCmd.Flags().StringVar(&cmd.ReleaseValues, "release-values", "", "DEPRECATED: use --extra-values instead")
cobraCmd.Flags().StringVar(&cmd.KubernetesVersion, "kubernetes-version", "", "The kubernetes version to use (e.g. v1.20). Patch versions are not supported")
cobraCmd.Flags().StringSliceVarP(&cmd.ExtraValues, "extra-values", "f", []string{}, "Path where to load extra helm values from")
cobraCmd.Flags().StringArrayVarP(&cmd.Values, "values", "f", []string{}, "Path where to load extra helm values from")
cobraCmd.Flags().StringArrayVar(&cmd.SetValues, "set", []string{}, "Set values for helm. E.g. --set 'persistence.enabled=true'")
cobraCmd.Flags().StringSliceVar(&cmd.DeprecatedExtraValues, "values", []string{}, "DEPRECATED: use --values instead")

cobraCmd.Flags().BoolVar(&cmd.CreateNamespace, "create-namespace", true, "If true the namespace will be created if it does not exist")
cobraCmd.Flags().BoolVar(&cmd.DisableIngressSync, "disable-ingress-sync", false, "If true the virtual cluster will not sync any ingresses")
cobraCmd.Flags().BoolVar(&cmd.UpdateCurrent, "update-current", true, "If true updates the current kube config")
cobraCmd.Flags().BoolVar(&cmd.CreateClusterRole, "create-cluster-role", false, "DEPRECATED: cluster role is now automatically created if it is required by one of the resource syncers that are enabled by the .sync.RESOURCE.enabled=true helm value, which is set in a file that is passed via --extra-values argument.")
cobraCmd.Flags().BoolVar(&cmd.Expose, "expose", false, "If true will create a load balancer service to expose the vcluster endpoint")
cobraCmd.Flags().BoolVar(&cmd.ExposeLocal, "expose-local", true, "If true and a local Kubernetes distro is detected, will deploy vcluster with a NodePort service. Will be set to false and the passed value will be ignored if --expose is set to true.")

cobraCmd.Flags().BoolVar(&cmd.Connect, "connect", true, "If true will run vcluster connect directly after the vcluster was created")
cobraCmd.Flags().BoolVar(&cmd.Upgrade, "upgrade", false, "If true will try to upgrade the vcluster instead of failing if it already exists")
cobraCmd.Flags().BoolVar(&cmd.Isolate, "isolate", false, "If true vcluster and its workloads will run in an isolated environment")
cobraCmd.Flags().BoolVar(&cmd.DisablePro, "disable-pro", false, "If true vcluster will not try to create a vCluster.Pro. You can also use 'vcluster logout' to prevent vCluster from creating any pro clusters")

// pro flags
cobraCmd.Flags().StringVar(&cmd.Project, "project", "", "[PRO] The vCluster.Pro project to use")
cobraCmd.Flags().StringVar(&cmd.Cluster, "cluster", "", "[PRO] The vCluster.Pro connected cluster to use")
cobraCmd.Flags().StringVar(&cmd.Template, "template", "", "[PRO] The vCluster.Pro template to use")
cobraCmd.Flags().StringVar(&cmd.TemplateVersion, "template-version", "", "[PRO] The vCluster.Pro template version to use")
cobraCmd.Flags().StringArrayVar(&cmd.Links, "link", []string{}, "[PRO] A link to add to the vCluster. E.g. --link 'prod=http://exampleprod.com'")
cobraCmd.Flags().StringVar(&cmd.Params, "params", "", "[PRO] If a template is used, this can be used to use a file for the parameters. E.g. --params path/to/my/file.yaml")
cobraCmd.Flags().StringArrayVar(&cmd.SetParams, "set-param", []string{}, "[PRO] If a template is used, this can be used to set a specific parameter. E.g. --set-param 'my-param=my-value'")

// hidden / deprecated
cobraCmd.Flags().StringVar(&cmd.LocalChartDir, "local-chart-dir", "", "The virtual cluster local chart dir to use")
cobraCmd.Flags().BoolVar(&cmd.DisableIngressSync, "disable-ingress-sync", false, "DEPRECATED: use --set 'sync.ingresses.enabled=false'")
cobraCmd.Flags().BoolVar(&cmd.ExposeLocal, "expose-local", true, "If true and a local Kubernetes distro is detected, will deploy vcluster with a NodePort service. Will be set to false and the passed value will be ignored if --expose is set to true.")

_ = cobraCmd.Flags().MarkHidden("local-chart-dir")
_ = cobraCmd.Flags().MarkHidden("disable-ingress-sync")
_ = cobraCmd.Flags().MarkHidden("expose-local")
return cobraCmd
}

func validateDeprecated(createOptions *create.Options, log log.Logger) {
if createOptions.ReleaseValues != "" {
log.Warn("Flag --release-values is deprecated, please use --extra-values instead. This flag will be removed in future!")
var loginText = "\nPlease run:\n * 'vcluster login' to connect to an existing vCluster.Pro instance\n * 'vcluster pro start' to deploy a new vCluster.Pro instance"

func (cmd *CreateCmd) validateOSSFlags() error {
if cmd.Project != "" {
return fmt.Errorf("cannot use --project as you are not connected to a vCluster.Pro instance." + loginText)
}
if cmd.Cluster != "" {
return fmt.Errorf("cannot use --cluster as you are not connected to a vCluster.Pro instance." + loginText)
}
if cmd.Template != "" {
return fmt.Errorf("cannot use --template as you are not connected to a vCluster.Pro instance." + loginText)
}
if createOptions.K3SImage != "" {
log.Warn("Flag --k3s-image is deprecated, please use --extra-values instead. This flag will be removed in future!")
if cmd.TemplateVersion != "" {
return fmt.Errorf("cannot use --template-version as you are not connected to a vCluster.Pro instance." + loginText)
}
if createOptions.CreateClusterRole {
log.Warn("Flag --create-cluster-role is deprecated. Cluster role is now automatically created if it is required by one of the resource syncers that are enabled by the .sync.RESOURCE.enabled=true helm value, which is set in a file that is passed via --extra-values (or -f) argument.")
if len(cmd.Links) > 0 {
return fmt.Errorf("cannot use --link as you are not connected to a vCluster.Pro instance." + loginText)
}
if cmd.Params != "" {
return fmt.Errorf("cannot use --params as you are not connected to a vCluster.Pro instance." + loginText)
}
if len(cmd.SetParams) > 0 {
return fmt.Errorf("cannot use --set-params as you are not connected to a vCluster.Pro instance." + loginText)
}

return nil
}

// Run executes the functionality
func (cmd *CreateCmd) Run(ctx context.Context, args []string) error {
cmd.Values = append(cmd.Values, cmd.DeprecatedExtraValues...)

// check if we should create a pro cluster
if !cmd.DisablePro {
proClient, err := pro.CreateProClient()
if err == nil {
return create.DeployProCluster(ctx, &cmd.Options, proClient, args[0], cmd.log)
}
}

// validate oss flags
err := cmd.validateOSSFlags()
if err != nil {
return err
}

// check helm binary
helmBinaryPath, err := GetHelmBinaryPath(ctx, cmd.log)
if err != nil {
return err
Expand All @@ -138,8 +189,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error {
output, err := exec.Command(helmBinaryPath, "version", "--client").CombinedOutput()
if errHelm := CheckHelmVersion(string(output)); errHelm != nil {
return errHelm
}
if err != nil {
} else if err != nil {
return err
}

Expand All @@ -166,7 +216,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error {
}

var newExtraValues []string
for _, value := range cmd.ExtraValues {
for _, value := range cmd.Values {
decodedString, err := getBase64DecodedString(value)
// ignore decoding errors and treat it as non-base64 string
if err != nil {
Expand Down Expand Up @@ -198,11 +248,7 @@ func (cmd *CreateCmd) Run(ctx context.Context, args []string) error {
}

// resetting this as the base64 encoded strings should be removed and only valid file names should be kept.
cmd.ExtraValues = newExtraValues

if cmd.ReleaseValues != "" {
cmd.ExtraValues = append(cmd.ExtraValues, cmd.ReleaseValues)
}
cmd.Values = newExtraValues

// check if vcluster already exists
if !cmd.Upgrade {
Expand Down Expand Up @@ -325,7 +371,8 @@ func (cmd *CreateCmd) deployChart(ctx context.Context, vClusterName, chartValues
Repo: cmd.ChartRepo,
Version: cmd.ChartVersion,
Values: chartValues,
ValuesFiles: cmd.ExtraValues,
ValuesFiles: cmd.Values,
SetValues: cmd.SetValues,
})
if err != nil {
return err
Expand Down Expand Up @@ -371,12 +418,11 @@ func (cmd *CreateCmd) ToChartOptions(kubernetesVersion *version.Info) (*helmUtil
ChartRepo: cmd.ChartRepo,
ChartVersion: cmd.ChartVersion,
CIDR: cmd.CIDR,
CreateClusterRole: cmd.CreateClusterRole,
CreateClusterRole: true,
DisableIngressSync: cmd.DisableIngressSync,
Expose: cmd.Expose,
SyncNodes: cmd.localCluster,
NodePort: cmd.localCluster,
K3SImage: cmd.K3SImage,
Isolate: cmd.Isolate,
KubernetesVersion: helmUtils.Version{
Major: kubernetesVersion.Major,
Expand Down
Loading

0 comments on commit 27ef31b

Please sign in to comment.