Use custom datadir in kubeconfig command (#536)

* Use custom datadir in kubeconfig command Signed-off-by: Kimmo Lehto <[email protected]> * Dummy change to trigger build Signed-off-by: Kimmo Lehto <[email protected]> * This is getting weird Signed-off-by: Kimmo Lehto <[email protected]> --------- Signed-off-by: Kimmo Lehto <[email protected]>
k0sproject · Aug 16, 2023 · 5e11c56 · 5e11c56
1 parent 8e05858
commit 5e11c56
Show file tree

Hide file tree

Showing 14 changed files with 46 additions and 41 deletions.
diff --git a/cmd/config_edit.go b/cmd/config_edit.go
@@ -65,7 +65,7 @@ var configEditCommand = &cli.Command{
 			return err
 		}
 
-		oldCfg, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get clusterconfig k0s -o yaml", h.DataDir), exec.Sudo(h))
+		oldCfg, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get clusterconfig k0s -o yaml", h.K0sDataDir()), exec.Sudo(h))
 		if err != nil {
 			return fmt.Errorf("%s: %w", h, err)
 		}
@@ -102,7 +102,7 @@ var configEditCommand = &cli.Command{
 			return fmt.Errorf("configuration was not changed, aborting")
 		}
 
-		if err := h.Exec(h.Configurer.K0sCmdf("kubectl apply --data-dir=%s -n kube-system -f -", h.DataDir), exec.Stdin(newCfg), exec.Sudo(h)); err != nil {
+		if err := h.Exec(h.Configurer.K0sCmdf("kubectl apply --data-dir=%s -n kube-system -f -", h.K0sDataDir()), exec.Stdin(newCfg), exec.Sudo(h)); err != nil {
 			return err
 		}
 

diff --git a/cmd/config_status.go b/cmd/config_status.go
@@ -47,7 +47,7 @@ var configStatusCommand = &cli.Command{
 			format = "-o " + format
 		}
 
-		output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s %s", h.DataDir, format), exec.Sudo(h))
+		output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s %s", h.K0sDataDir(), format), exec.Sudo(h))
 		if err != nil {
 			return fmt.Errorf("%s: %w", h, err)
 		}

diff --git a/configurer/linux.go b/configurer/linux.go
@@ -2,6 +2,7 @@ package configurer
 
 import (
 	"fmt"
+	"path"
 	"regexp"
 	"strconv"
 	"strings"
@@ -17,7 +18,7 @@ type PathFuncs interface {
 	K0sBinaryPath() string
 	K0sConfigPath() string
 	K0sJoinTokenPath() string
-	KubeconfigPath(h os.Host) string
+	KubeconfigPath(os.Host, string) string
 	DataDirDefaultPath() string
 }
 
@@ -137,12 +138,15 @@ func (l Linux) MoveFile(h os.Host, src, dst string) error {
 }
 
 // KubeconfigPath returns the path to a kubeconfig on the host
-func (l Linux) KubeconfigPath(h os.Host) string {
+func (l Linux) KubeconfigPath(h os.Host, dataDir string) string {
 	linux := &os.Linux{}
-	if linux.FileExist(h, "/var/lib/k0s/pki/admin.conf") {
-		return "/var/lib/k0s/pki/admin.conf"
+
+	// if admin.conf exists, use that
+	adminConfPath := path.Join(dataDir, "pki/admin.conf")
+	if linux.FileExist(h, adminConfPath) {
+		return adminConfPath
 	}
-	return "/var/lib/k0s/kubelet.conf"
+	return path.Join(dataDir, "kubelet.conf")
 }
 
 // DataDirPath returns the location of k0s data dir
@@ -151,8 +155,8 @@ func (l Linux) DataDirDefaultPath() string {
 }
 
 // KubectlCmdf returns a command line in sprintf manner for running kubectl on the host using the kubeconfig from KubeconfigPath
-func (l Linux) KubectlCmdf(h os.Host, s string, args ...interface{}) string {
-	return fmt.Sprintf(`env "KUBECONFIG=%s" %s`, l.PathFuncs.KubeconfigPath(h), l.K0sCmdf(`kubectl %s`, fmt.Sprintf(s, args...)))
+func (l Linux) KubectlCmdf(h os.Host, dataDir, s string, args ...interface{}) string {
+	return fmt.Sprintf(`env "KUBECONFIG=%s" %s`, l.PathFuncs.KubeconfigPath(h, dataDir), l.K0sCmdf(`kubectl %s`, fmt.Sprintf(s, args...)))
 }
 
 // HTTPStatus makes a HTTP GET request to the url and returns the status code or an error

diff --git a/configurer/linux/linux_test.go b/configurer/linux/linux_test.go
@@ -65,9 +65,9 @@ func TestPaths(t *testing.T) {
 	require.Equal(t, "/opt/bin/k0s --help", fc.K0sCmdf("--help"))
 	require.Equal(t, "/usr/local/bin/k0s --help", ubuntu.K0sCmdf("--help"))
 
-	require.Equal(t, "/var/lib/k0s/pki/admin.conf", fc.KubeconfigPath(h1))
-	require.Equal(t, "/var/lib/k0s/pki/admin.conf", ubuntu.KubeconfigPath(h1))
+	require.Equal(t, "/var/lib/k0s/pki/admin.conf", fc.KubeconfigPath(h1, fc.DataDirDefaultPath()))
+	require.Equal(t, "/var/lib/k0s/pki/admin.conf", ubuntu.KubeconfigPath(h1, ubuntu.DataDirDefaultPath()))
 
-	require.Equal(t, "/var/lib/k0s/kubelet.conf", fc.KubeconfigPath(h2))
-	require.Equal(t, "/var/lib/k0s/kubelet.conf", ubuntu.KubeconfigPath(h2))
+	require.Equal(t, "/var/lib/k0s/kubelet.conf", fc.KubeconfigPath(h2, fc.DataDirDefaultPath()))
+	require.Equal(t, "/var/lib/k0s/kubelet.conf", ubuntu.KubeconfigPath(h2, ubuntu.DataDirDefaultPath()))
 }
diff --git a/phase/configure_k0s.go b/phase/configure_k0s.go
@@ -41,7 +41,7 @@ func (p *ConfigureK0s) Run() error {
 
 		var cmd string
 		if p.leader.Exec(p.leader.Configurer.K0sCmdf("config create --help"), exec.Sudo(p.leader)) == nil {
-			cmd = p.leader.Configurer.K0sCmdf("config create --data-dir=%s", p.leader.DataDir)
+			cmd = p.leader.Configurer.K0sCmdf("config create --data-dir=%s", p.leader.K0sDataDir())
 		} else {
 			cmd = p.leader.Configurer.K0sCmdf("default-config")
 		}

diff --git a/phase/get_kubeconfig.go b/phase/get_kubeconfig.go
@@ -20,7 +20,7 @@ func (p *GetKubeconfig) Title() string {
 }
 
 var readKubeconfig = func(h *cluster.Host) (string, error) {
-	return h.Configurer.ReadFile(h, h.Configurer.KubeconfigPath(h))
+	return h.Configurer.ReadFile(h, h.Configurer.KubeconfigPath(h, h.K0sDataDir()))
 }
 
 // Run the phase

diff --git a/phase/install_controllers.go b/phase/install_controllers.go
@@ -67,7 +67,7 @@ func (p *InstallControllers) Run() error {
 		}
 		log.Debugf("%s: join token ID: %s", p.leader, tokenID)
 		defer func() {
-			if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", h.DataDir, tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil {
+			if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", h.K0sDataDir(), tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil {
 				log.Warnf("%s: failed to invalidate the controller join token", p.leader)
 			}
 		}()

diff --git a/phase/install_workers.go b/phase/install_workers.go
@@ -85,7 +85,7 @@ func (p *InstallWorkers) Run() error {
 
 	if !NoWait {
 		defer func() {
-			if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", p.leader.DataDir, tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil {
+			if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", p.leader.K0sDataDir(), tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil {
 				log.Warnf("%s: failed to invalidate the worker join token", p.leader)
 			}
 		}()

diff --git a/phase/prepare_hosts.go b/phase/prepare_hosts.go
@@ -3,7 +3,6 @@ package phase
 import (
 	"strings"
 
-	"github.com/alessio/shellescape"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/rig/os"
 	log "github.com/sirupsen/logrus"
@@ -70,11 +69,5 @@ func (p *PrepareHosts) prepareHost(h *cluster.Host) error {
 		}
 	}
 
-	if h.DataDir == "" {
-		log.Debugf("%s: data-dir is not set, using default", h)
-		h.DataDir = h.Configurer.DataDirDefaultPath()
-	}
-	h.DataDir = shellescape.Quote(h.DataDir)
-
 	return nil
 }
diff --git a/phase/reset_controllers.go b/phase/reset_controllers.go
@@ -103,7 +103,7 @@ func (p *ResetControllers) Run() error {
 		log.Debugf("%s: leaving etcd completed", h)
 
 		log.Debugf("%s: resetting k0s...", h)
-		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.DataDir), exec.Sudo(h))
+		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h))
 		if err != nil {
 			log.Debugf("%s: k0s reset failed: %s", h, out)
 			log.Warnf("%s: k0s reported failure: %v", h, err)

diff --git a/phase/reset_leader.go b/phase/reset_leader.go
@@ -49,7 +49,7 @@ func (p *ResetLeader) Run() error {
 	}
 
 	log.Debugf("%s: resetting k0s...", p.leader)
-	out, err := p.leader.ExecOutput(p.leader.Configurer.K0sCmdf("reset --data-dir=%s", p.leader.DataDir), exec.Sudo(p.leader))
+	out, err := p.leader.ExecOutput(p.leader.Configurer.K0sCmdf("reset --data-dir=%s", p.leader.K0sDataDir()), exec.Sudo(p.leader))
 	if err != nil {
 		log.Debugf("%s: k0s reset failed: %s", p.leader, out)
 		log.Warnf("%s: k0s reported failure: %v", p.leader, err)

diff --git a/phase/reset_workers.go b/phase/reset_workers.go
@@ -94,7 +94,7 @@ func (p *ResetWorkers) Run() error {
 		}
 
 		log.Debugf("%s: resetting k0s...", h)
-		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.DataDir), exec.Sudo(h))
+		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h))
 		if err != nil {
 			log.Debugf("%s: k0s reset failed: %s", h, out)
 			log.Warnf("%s: k0s reported failure: %v", h, err)

diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go
@@ -125,8 +125,8 @@ type configurer interface {
 	DeleteFile(os.Host, string) error
 	CommandExist(os.Host, string) bool
 	Hostname(os.Host) string
-	KubectlCmdf(os.Host, string, ...interface{}) string
-	KubeconfigPath(os.Host) string
+	KubectlCmdf(os.Host, string, string, ...interface{}) string
+	KubeconfigPath(os.Host, string) string
 	IsContainer(os.Host) bool
 	FixContainer(os.Host) error
 	HTTPStatus(os.Host, string) (int, error)
@@ -260,7 +260,7 @@ func (h *Host) K0sInstallCommand() (string, error) {
 	role := h.Role
 	flags := h.InstallFlags
 
-	flags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.DataDir))
+	flags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.K0sDataDir()))
 
 	switch role {
 	case "controller+worker":
@@ -330,12 +330,12 @@ func (h *Host) K0sInstallCommand() (string, error) {
 
 // K0sBackupCommand returns a full command to be used as run k0s backup
 func (h *Host) K0sBackupCommand(targetDir string) string {
-	return h.Configurer.K0sCmdf("backup --save-path %s --data-dir %s", shellescape.Quote(targetDir), h.DataDir)
+	return h.Configurer.K0sCmdf("backup --save-path %s --data-dir %s", shellescape.Quote(targetDir), h.K0sDataDir())
 }
 
 // K0sRestoreCommand returns a full command to restore cluster state from a backup
 func (h *Host) K0sRestoreCommand(backupfile string) string {
-	return h.Configurer.K0sCmdf("restore --data-dir=%s %s", h.DataDir, shellescape.Quote(backupfile))
+	return h.Configurer.K0sCmdf("restore --data-dir=%s %s", h.K0sDataDir(), shellescape.Quote(backupfile))
 }
 
 // IsController returns true for controller and controller+worker roles
@@ -390,6 +390,14 @@ func (h *Host) UpdateK0sBinary(path string, version *version.Version) error {
 	return nil
 }
 
+// K0sDataDir returns the data dir for the host either from host.DataDir or the default from configurer's DataDirDefaultPath
+func (h *Host) K0sDataDir() string {
+	if h.DataDir == "" {
+		return h.Configurer.DataDirDefaultPath()
+	}
+	return h.DataDir
+}
+
 type kubeNodeStatus struct {
 	Items []struct {
 		Status struct {
@@ -403,7 +411,7 @@ type kubeNodeStatus struct {
 
 // KubeNodeReady runs kubectl on the host and returns true if the given node is marked as ready
 func (h *Host) KubeNodeReady() (bool, error) {
-	output, err := h.ExecOutput(h.Configurer.KubectlCmdf(h, "get node --data-dir=%s -l kubernetes.io/hostname=%s -o json", h.DataDir, h.Metadata.Hostname), exec.HideOutput(), exec.Sudo(h))
+	output, err := h.ExecOutput(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "get node -l kubernetes.io/hostname=%s -o json", h.Metadata.Hostname), exec.HideOutput(), exec.Sudo(h))
 	if err != nil {
 		return false, err
 	}
@@ -456,7 +464,7 @@ type statusEvents struct {
 func (h *Host) WaitK0sDynamicConfigReady() error {
 	return retry.Do(
 		func() error {
-			output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s -o json", h.DataDir), exec.Sudo(h))
+			output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s -o json", h.K0sDataDir()), exec.Sudo(h))
 			if err != nil {
 				return fmt.Errorf("failed to get k0s config status events: %w", err)
 			}
@@ -481,25 +489,25 @@ func (h *Host) WaitK0sDynamicConfigReady() error {
 
 // DrainNode drains the given node
 func (h *Host) DrainNode(node *Host) error {
-	return h.Exec(h.Configurer.KubectlCmdf(h, "drain --data-dir=%s --grace-period=120 --force --timeout=5m --ignore-daemonsets --delete-emptydir-data %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h))
+	return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "drain --grace-period=120 --force --timeout=5m --ignore-daemonsets --delete-emptydir-data %s", node.Metadata.Hostname), exec.Sudo(h))
 }
 
 // UncordonNode marks the node schedulable again
 func (h *Host) UncordonNode(node *Host) error {
-	return h.Exec(h.Configurer.KubectlCmdf(h, "uncordon --data-dir=%s %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h))
+	return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "uncordon %s", node.Metadata.Hostname), exec.Sudo(h))
 }
 
 // DeleteNode deletes the given node from kubernetes
 func (h *Host) DeleteNode(node *Host) error {
-	return h.Exec(h.Configurer.KubectlCmdf(h, "delete node --data-dir=%s %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h))
+	return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "delete node %s", node.Metadata.Hostname), exec.Sudo(h))
 }
 
 func (h *Host) LeaveEtcd(node *Host) error {
 	etcdAddress := node.SSH.Address
 	if node.PrivateAddress != "" {
 		etcdAddress = node.PrivateAddress
 	}
-	return h.Exec(h.Configurer.K0sCmdf("etcd leave --peer-address %s --datadir %s", etcdAddress, h.DataDir), exec.Sudo(h))
+	return h.Exec(h.Configurer.K0sCmdf("etcd leave --peer-address %s --datadir %s", etcdAddress, h.K0sDataDir()), exec.Sudo(h))
 }
 
 // CheckHTTPStatus will perform a web request to the url and return an error if the http status is not the expected

diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go
@@ -144,7 +144,7 @@ func (k K0s) GenerateToken(h *Host, role string, expiry time.Duration) (string,
 		k0sFlags.Add(fmt.Sprintf("--config %s", shellescape.Quote(h.K0sConfigPath())))
 	}
 
-	k0sFlags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.DataDir))
+	k0sFlags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.K0sDataDir()))
 
 	var token string
 	err = retry.Do(
@@ -167,7 +167,7 @@ func (k K0s) GenerateToken(h *Host, role string, expiry time.Duration) (string,
 
 // GetClusterID uses kubectl to fetch the kube-system namespace uid
 func (k K0s) GetClusterID(h *Host) (string, error) {
-	return h.ExecOutput(h.Configurer.KubectlCmdf(h, "get --data-dir=%s -n kube-system namespace kube-system -o template={{.metadata.uid}}", h.DataDir), exec.Sudo(h))
+	return h.ExecOutput(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "get -n kube-system namespace kube-system -o template={{.metadata.uid}}"), exec.Sudo(h))
 }
 
 // VersionEqual returns true if the configured k0s version is equal to the given version string