From a7ef4b4f783adbb8ae3f5fa3434a603d32296b25 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Tue, 18 Jun 2024 17:12:16 +0200 Subject: [PATCH] fix: background-proxy flag --- cmd/vclusterctl/cmd/logout.go | 11 +-- pkg/cli/connect_helm.go | 25 +++--- pkg/cli/create_helm.go | 3 + pkg/cli/flags/connect/connect.go | 2 +- pkg/cli/flags/create/create.go | 1 + pkg/cli/localkubernetes/configure.go | 46 +++++------ pkg/util/kubeconfig/kubeconfig.go | 114 +++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 42 deletions(-) diff --git a/cmd/vclusterctl/cmd/logout.go b/cmd/vclusterctl/cmd/logout.go index 8ec752666..2b6e7de1e 100644 --- a/cmd/vclusterctl/cmd/logout.go +++ b/cmd/vclusterctl/cmd/logout.go @@ -10,7 +10,6 @@ import ( "github.com/loft-sh/vcluster/pkg/cli/config" "github.com/loft-sh/vcluster/pkg/cli/flags" "github.com/loft-sh/vcluster/pkg/platform" - "github.com/loft-sh/vcluster/pkg/upgrade" "github.com/mgutz/ansi" "github.com/spf13/cobra" ) @@ -43,9 +42,6 @@ vcluster logout Long: description, Args: cobra.NoArgs, RunE: func(cobraCmd *cobra.Command, _ []string) error { - // Check for newer version - upgrade.PrintNewerVersionWarning() - return cmd.Run(cobraCmd.Context()) }, } @@ -56,15 +52,14 @@ vcluster logout func (cmd *LogoutCmd) Run(ctx context.Context) error { platformClient := platform.NewClientFromConfig(cmd.LoadedConfig(cmd.Log)) - cfg := platformClient.Config() - // delete old access key if were logged in before + cfg := platformClient.Config() if cfg.Platform.AccessKey != "" { if err := platformClient.Logout(ctx); err != nil { - return fmt.Errorf("failed to logout: %w", err) + cmd.Log.Errorf("failed to send logout request: %v", err) } - configHost := cfg.Platform.Host + configHost := cfg.Platform.Host cfg.Platform.Host = "" cfg.Platform.AccessKey = "" cfg.Platform.LastInstallContext = "" diff --git a/pkg/cli/connect_helm.go b/pkg/cli/connect_helm.go index 1b24ad683..5d13101b8 100644 --- a/pkg/cli/connect_helm.go +++ b/pkg/cli/connect_helm.go @@ -100,16 +100,6 @@ func (cmd *connectHelm) connect(ctx context.Context, vCluster *find.VCluster, co return err } - if len(command) == 0 && cmd.ServiceAccount == "" && cmd.Server == "" && cmd.BackgroundProxy && localkubernetes.IsDockerInstalledAndUpAndRunning() { - // start background container - server, err := localkubernetes.CreateBackgroundProxyContainer(ctx, vCluster.Name, cmd.Namespace, &cmd.rawConfig, kubeConfig, cmd.LocalPort, cmd.Log) - if err != nil { - cmd.Log.Warnf("Error exposing local vcluster, will fallback to port-forwarding: %v", err) - cmd.BackgroundProxy = false - } - cmd.Server = server - } - // check if we should execute command if len(command) > 0 { if !cmd.portForwarding { @@ -327,6 +317,21 @@ func (cmd *connectHelm) getVClusterKubeConfig(ctx context.Context, vclusterName if err != nil { return nil, err } + + // check if we should start a background proxy + if cmd.Server == "" && cmd.BackgroundProxy { + if localkubernetes.IsDockerInstalledAndUpAndRunning() { + // start background container + server, err := localkubernetes.CreateBackgroundProxyContainer(ctx, vclusterName, cmd.Namespace, cmd.kubeClientConfig, kubeConfig, cmd.LocalPort, cmd.Log) + if err != nil { + cmd.Log.Warnf("Error exposing local vcluster, will fallback to port-forwarding: %v", err) + cmd.BackgroundProxy = false + } + cmd.Server = server + } else { + cmd.Log.Debugf("Docker is not installed, so skip background proxy") + } + } } // find out vcluster server port diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 254bc0a72..b4d5d76f6 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -59,6 +59,7 @@ type CreateOptions struct { CreateNamespace bool UpdateCurrent bool + BackgroundProxy bool CreateContext bool SwitchContext bool Expose bool @@ -145,6 +146,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. if isVClusterDeployed(release) { if cmd.Connect { return ConnectHelm(ctx, &ConnectOptions{ + BackgroundProxy: cmd.BackgroundProxy, UpdateCurrent: cmd.UpdateCurrent, KubeConfigContextName: cmd.KubeConfigContextName, KubeConfig: "./kubeconfig.yaml", @@ -287,6 +289,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. if cmd.Connect || cmd.Print { cmd.log.Donef("Successfully created virtual cluster %s in namespace %s", vClusterName, cmd.Namespace) return ConnectHelm(ctx, &ConnectOptions{ + BackgroundProxy: cmd.BackgroundProxy, UpdateCurrent: cmd.UpdateCurrent, Print: cmd.Print, KubeConfigContextName: cmd.KubeConfigContextName, diff --git a/pkg/cli/flags/connect/connect.go b/pkg/cli/flags/connect/connect.go index a6b4d1e94..2a1aed0e2 100644 --- a/pkg/cli/flags/connect/connect.go +++ b/pkg/cli/flags/connect/connect.go @@ -21,7 +21,7 @@ func AddCommonFlags(cmd *cobra.Command, options *cli.ConnectOptions) { cmd.Flags().StringVar(&options.ServiceAccountClusterRole, "cluster-role", "", "If specified, vCluster will create the service account if it does not exist and also add a cluster role binding for the given cluster role to it. Requires --service-account to be set") cmd.Flags().IntVar(&options.ServiceAccountExpiration, "token-expiration", 0, "If specified, vCluster will create the service account token for the given duration in seconds. Defaults to eternal") cmd.Flags().BoolVar(&options.Insecure, "insecure", false, "If specified, vCluster will create the kube config with insecure-skip-tls-verify") - cmd.Flags().BoolVar(&options.BackgroundProxy, "background-proxy", false, "If specified, vCluster will create the background proxy in docker [its mainly used for vclusters with no nodeport service.]") + cmd.Flags().BoolVar(&options.BackgroundProxy, "background-proxy", true, "Try to use a background-proxy to access the vCluster. Only works if docker is installed and reachable") // deprecated _ = cmd.Flags().MarkDeprecated("kube-config", fmt.Sprintf("please use %q to write the kubeconfig of the virtual cluster to stdout.", "vcluster connect --print")) diff --git a/pkg/cli/flags/create/create.go b/pkg/cli/flags/create/create.go index 403edf4e6..7e9ba8ca0 100644 --- a/pkg/cli/flags/create/create.go +++ b/pkg/cli/flags/create/create.go @@ -35,6 +35,7 @@ func AddHelmFlags(cmd *cobra.Command, options *cli.CreateOptions) { cmd.Flags().BoolVar(&options.CreateNamespace, "create-namespace", true, "If true the namespace will be created if it does not exist") cmd.Flags().StringVar(&options.LocalChartDir, "local-chart-dir", "", "The virtual cluster local chart dir to use") cmd.Flags().BoolVar(&options.ExposeLocal, "expose-local", true, "If true and a local Kubernetes distro is detected, will deploy vcluster with a NodePort service. Will be set to false and the passed value will be ignored if --expose is set to true.") + cmd.Flags().BoolVar(&options.BackgroundProxy, "background-proxy", true, "Try to use a background-proxy to access the vCluster. Only works if docker is installed and reachable") _ = cmd.Flags().MarkHidden("local-chart-dir") _ = cmd.Flags().MarkHidden("expose-local") diff --git a/pkg/cli/localkubernetes/configure.go b/pkg/cli/localkubernetes/configure.go index 4f3586aab..6629e76be 100644 --- a/pkg/cli/localkubernetes/configure.go +++ b/pkg/cli/localkubernetes/configure.go @@ -10,10 +10,9 @@ import ( "strings" "time" - "github.com/loft-sh/vcluster/pkg/upgrade" - "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/cli/find" + "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -294,20 +293,23 @@ func createProxyContainer(ctx context.Context, vClusterName, vClusterNamespace s return server, nil } -func CreateBackgroundProxyContainer(ctx context.Context, vClusterName, vClusterNamespace string, rawConfig, vRawConfig *clientcmdapi.Config, localPort int, log log.Logger) (string, error) { +func CreateBackgroundProxyContainer(ctx context.Context, vClusterName, vClusterNamespace string, rawConfig clientcmd.ClientConfig, vRawConfig *clientcmdapi.Config, localPort int, log log.Logger) (string, error) { + rawConfigObj, err := rawConfig.RawConfig() + if err != nil { + return "", err + } + // write kube config to buffer - physicalCluster, err := clientcmd.Write(*rawConfig) + physicalCluster, err := kubeconfig.ResolveKubeConfig(rawConfig) if err != nil { - return "", nil + return "", fmt.Errorf("resolve kube config: %w", err) } + // write a temporary kube file tempFile, err := os.CreateTemp("", "") if err != nil { return "", errors.Wrap(err, "create temp file") } - defer func(name string) { - _ = os.Remove(name) - }(tempFile.Name()) _, err = tempFile.Write(physicalCluster) if err != nil { return "", errors.Wrap(err, "write kube config to temp file") @@ -319,33 +321,30 @@ func CreateBackgroundProxyContainer(ctx context.Context, vClusterName, vClusterN kubeConfigPath := tempFile.Name() // construct proxy name - proxyName := find.VClusterConnectBackgroundProxyName(vClusterName, vClusterNamespace, rawConfig.CurrentContext) + proxyName := find.VClusterConnectBackgroundProxyName(vClusterName, vClusterNamespace, rawConfigObj.CurrentContext) // check if the background proxy container for this vcluster is running and then remove it. _ = CleanupBackgroundProxy(proxyName, log) - // docker run -d --network=host -v /root/.kube/config:/root/.kube/config ghcr.io/loft-sh/vcluster-cli vcluster connect vcluster -n vcluster --local-port 13300 + // build the command cmd := exec.Command( "docker", "run", "-d", - "-v", - fmt.Sprintf("%v:%v", kubeConfigPath, "/root/.kube/config"), + "-v", fmt.Sprintf("%v:%v", kubeConfigPath, "/kube-config"), fmt.Sprintf("--name=%s", proxyName), - fmt.Sprintf("--network=%s", "host"), - "ghcr.io/loft-sh/vcluster-cli"+upgrade.GetVersion(), - "vcluster", - "connect", - vClusterName, - "--local-port", - strconv.Itoa(localPort), - "-n", - vClusterNamespace, + "--network=host", + "bitnami/kubectl:1.29", + "port-forward", + "svc/"+vClusterName, + strconv.Itoa(localPort)+":443", + "--kubeconfig", "/kube-config", + "-n", vClusterNamespace, ) log.Infof("Starting background proxy container...") - out, err := cmd.Output() + out, err := cmd.CombinedOutput() if err != nil { - return "", errors.Errorf("error starting background proxy : %s %v", string(out), err) + return "", errors.Errorf("error starting background proxy: %s %v", string(out), err) } server := fmt.Sprintf("https://127.0.0.1:%v", localPort) waitErr := wait.PollUntilContextTimeout(ctx, time.Second, time.Second*60, true, func(ctx context.Context) (bool, error) { @@ -358,6 +357,7 @@ func CreateBackgroundProxyContainer(ctx context.Context, vClusterName, vClusterN if waitErr != nil { return "", fmt.Errorf("test connection: %w %w", waitErr, err) } + return server, nil } diff --git a/pkg/util/kubeconfig/kubeconfig.go b/pkg/util/kubeconfig/kubeconfig.go index e4d98e195..e9e567182 100644 --- a/pkg/util/kubeconfig/kubeconfig.go +++ b/pkg/util/kubeconfig/kubeconfig.go @@ -1,15 +1,22 @@ package kubeconfig import ( + "bytes" "context" "fmt" "os" + "os/exec" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/pkg/apis/clientauthentication" + "k8s.io/client-go/pkg/apis/clientauthentication/install" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -174,3 +181,110 @@ func ConvertRestConfigToClientConfig(config *rest.Config) (clientcmd.ClientConfi return clientcmd.NewDefaultClientConfig(*kubeConfig, &clientcmd.ConfigOverrides{}), nil } + +func ResolveKubeConfig(rawConfig clientcmd.ClientConfig) ([]byte, error) { + restConfig, err := rawConfig.ClientConfig() + if err != nil { + return nil, err + } + + // convert exec auth + if restConfig.ExecProvider != nil { + err = resolveExecCredentials(restConfig) + if err != nil { + return nil, fmt.Errorf("resolve exec credentials: %w", err) + } + } + if restConfig.AuthProvider != nil { + return nil, fmt.Errorf("auth provider is not supported") + } + + retConfig, err := ConvertRestConfigToClientConfig(restConfig) + if err != nil { + return nil, err + } + + retRawConfig, err := retConfig.RawConfig() + if err != nil { + return nil, err + } + + return clientcmd.Write(retRawConfig) +} + +func resolveExecCredentials(restConfig *rest.Config) error { + cred := &clientauthentication.ExecCredential{ + Spec: clientauthentication.ExecCredentialSpec{ + Interactive: false, + }, + } + + execProvider := restConfig.ExecProvider + if execProvider.ProvideClusterInfo { + var err error + cred.Spec.Cluster, err = rest.ConfigToExecCluster(restConfig) + if err != nil { + return err + } + } + + env := os.Environ() + for _, e := range execProvider.Env { + env = append(env, e.Name+"="+e.Value) + } + + groupVersion, err := schema.ParseGroupVersion(execProvider.APIVersion) + if err != nil { + return err + } + + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + install.Install(scheme) + data, err := runtime.Encode(codecs.LegacyCodec(groupVersion), cred) + if err != nil { + return fmt.Errorf("encode ExecCredentials: %w", err) + } + env = append(env, fmt.Sprintf("%s=%s", "KUBERNETES_EXEC_INFO", data)) + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + cmd := exec.Command(execProvider.Command, execProvider.Args...) + cmd.Env = env + cmd.Stderr = stderr + cmd.Stdout = stdout + + err = cmd.Run() + if err != nil { + return fmt.Errorf("error executing exec provider: %s %s %w", stderr.String(), stdout.String(), err) + } + + _, gvk, err := codecs.UniversalDecoder(groupVersion).Decode(stdout.Bytes(), nil, cred) + if err != nil { + return fmt.Errorf("decoding stdout: %w", err) + } + if gvk.Group != groupVersion.Group || gvk.Version != groupVersion.Version { + return fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s", + groupVersion, schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}) + } + + if cred.Status == nil { + return fmt.Errorf("exec plugin didn't return a status field") + } + if cred.Status.Token == "" && cred.Status.ClientCertificateData == "" && cred.Status.ClientKeyData == "" { + return fmt.Errorf("exec plugin didn't return a token or cert/key pair") + } + if (cred.Status.ClientCertificateData == "") != (cred.Status.ClientKeyData == "") { + return fmt.Errorf("exec plugin returned only certificate or key, not both") + } + + if cred.Status.Token != "" { + restConfig.BearerToken = cred.Status.Token + } else if cred.Status.ClientKeyData != "" && cred.Status.ClientCertificateData != "" { + restConfig.KeyData = []byte(cred.Status.ClientKeyData) + restConfig.CertData = []byte(cred.Status.ClientCertificateData) + } + + restConfig.ExecProvider = nil + return nil +}