diff --git a/cmd/config_edit.go b/cmd/config_edit.go index a92a61c7..f1fcb3e5 100644 --- a/cmd/config_edit.go +++ b/cmd/config_edit.go @@ -65,7 +65,7 @@ var configEditCommand = &cli.Command{ return err } - oldCfg, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get clusterconfig k0s -o yaml", h.DataDir), exec.Sudo(h)) + oldCfg, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get clusterconfig k0s -o yaml", h.K0sDataDir()), exec.Sudo(h)) if err != nil { return fmt.Errorf("%s: %w", h, err) } @@ -102,7 +102,7 @@ var configEditCommand = &cli.Command{ return fmt.Errorf("configuration was not changed, aborting") } - if err := h.Exec(h.Configurer.K0sCmdf("kubectl apply --data-dir=%s -n kube-system -f -", h.DataDir), exec.Stdin(newCfg), exec.Sudo(h)); err != nil { + if err := h.Exec(h.Configurer.K0sCmdf("kubectl apply --data-dir=%s -n kube-system -f -", h.K0sDataDir()), exec.Stdin(newCfg), exec.Sudo(h)); err != nil { return err } diff --git a/cmd/config_status.go b/cmd/config_status.go index fb64ee87..716a5083 100644 --- a/cmd/config_status.go +++ b/cmd/config_status.go @@ -47,7 +47,7 @@ var configStatusCommand = &cli.Command{ format = "-o " + format } - output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s %s", h.DataDir, format), exec.Sudo(h)) + output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s %s", h.K0sDataDir(), format), exec.Sudo(h)) if err != nil { return fmt.Errorf("%s: %w", h, err) } diff --git a/configurer/linux.go b/configurer/linux.go index a0bbdd61..82152a6a 100644 --- a/configurer/linux.go +++ b/configurer/linux.go @@ -2,6 +2,7 @@ package configurer import ( "fmt" + "path" "regexp" "strconv" "strings" @@ -17,7 +18,7 @@ type PathFuncs interface { K0sBinaryPath() string K0sConfigPath() string K0sJoinTokenPath() string - KubeconfigPath(h os.Host) string + KubeconfigPath(os.Host, string) string DataDirDefaultPath() string } @@ -137,12 +138,15 @@ func (l Linux) MoveFile(h os.Host, src, dst string) error { } // KubeconfigPath returns the path to a kubeconfig on the host -func (l Linux) KubeconfigPath(h os.Host) string { +func (l Linux) KubeconfigPath(h os.Host, dataDir string) string { linux := &os.Linux{} - if linux.FileExist(h, "/var/lib/k0s/pki/admin.conf") { - return "/var/lib/k0s/pki/admin.conf" + + // if admin.conf exists, use that + adminConfPath := path.Join(dataDir, "pki/admin.conf") + if linux.FileExist(h, adminConfPath) { + return adminConfPath } - return "/var/lib/k0s/kubelet.conf" + return path.Join(dataDir, "kubelet.conf") } // DataDirPath returns the location of k0s data dir @@ -151,8 +155,8 @@ func (l Linux) DataDirDefaultPath() string { } // KubectlCmdf returns a command line in sprintf manner for running kubectl on the host using the kubeconfig from KubeconfigPath -func (l Linux) KubectlCmdf(h os.Host, s string, args ...interface{}) string { - return fmt.Sprintf(`env "KUBECONFIG=%s" %s`, l.PathFuncs.KubeconfigPath(h), l.K0sCmdf(`kubectl %s`, fmt.Sprintf(s, args...))) +func (l Linux) KubectlCmdf(h os.Host, dataDir, s string, args ...interface{}) string { + return fmt.Sprintf(`env "KUBECONFIG=%s" %s`, l.PathFuncs.KubeconfigPath(h, dataDir), l.K0sCmdf(`kubectl %s`, fmt.Sprintf(s, args...))) } // HTTPStatus makes a HTTP GET request to the url and returns the status code or an error diff --git a/configurer/linux/linux_test.go b/configurer/linux/linux_test.go index ac20b338..2c0a46cf 100644 --- a/configurer/linux/linux_test.go +++ b/configurer/linux/linux_test.go @@ -65,9 +65,9 @@ func TestPaths(t *testing.T) { require.Equal(t, "/opt/bin/k0s --help", fc.K0sCmdf("--help")) require.Equal(t, "/usr/local/bin/k0s --help", ubuntu.K0sCmdf("--help")) - require.Equal(t, "/var/lib/k0s/pki/admin.conf", fc.KubeconfigPath(h1)) - require.Equal(t, "/var/lib/k0s/pki/admin.conf", ubuntu.KubeconfigPath(h1)) + require.Equal(t, "/var/lib/k0s/pki/admin.conf", fc.KubeconfigPath(h1, fc.DataDirDefaultPath())) + require.Equal(t, "/var/lib/k0s/pki/admin.conf", ubuntu.KubeconfigPath(h1, ubuntu.DataDirDefaultPath())) - require.Equal(t, "/var/lib/k0s/kubelet.conf", fc.KubeconfigPath(h2)) - require.Equal(t, "/var/lib/k0s/kubelet.conf", ubuntu.KubeconfigPath(h2)) + require.Equal(t, "/var/lib/k0s/kubelet.conf", fc.KubeconfigPath(h2, fc.DataDirDefaultPath())) + require.Equal(t, "/var/lib/k0s/kubelet.conf", ubuntu.KubeconfigPath(h2, ubuntu.DataDirDefaultPath())) } diff --git a/phase/configure_k0s.go b/phase/configure_k0s.go index c5cea942..2230b18e 100644 --- a/phase/configure_k0s.go +++ b/phase/configure_k0s.go @@ -41,7 +41,7 @@ func (p *ConfigureK0s) Run() error { var cmd string if p.leader.Exec(p.leader.Configurer.K0sCmdf("config create --help"), exec.Sudo(p.leader)) == nil { - cmd = p.leader.Configurer.K0sCmdf("config create --data-dir=%s", p.leader.DataDir) + cmd = p.leader.Configurer.K0sCmdf("config create --data-dir=%s", p.leader.K0sDataDir()) } else { cmd = p.leader.Configurer.K0sCmdf("default-config") } diff --git a/phase/get_kubeconfig.go b/phase/get_kubeconfig.go index bb63a360..90ee5dbe 100644 --- a/phase/get_kubeconfig.go +++ b/phase/get_kubeconfig.go @@ -20,7 +20,7 @@ func (p *GetKubeconfig) Title() string { } var readKubeconfig = func(h *cluster.Host) (string, error) { - return h.Configurer.ReadFile(h, h.Configurer.KubeconfigPath(h)) + return h.Configurer.ReadFile(h, h.Configurer.KubeconfigPath(h, h.K0sDataDir())) } // Run the phase diff --git a/phase/install_controllers.go b/phase/install_controllers.go index 7a7fd042..8219c819 100644 --- a/phase/install_controllers.go +++ b/phase/install_controllers.go @@ -67,7 +67,7 @@ func (p *InstallControllers) Run() error { } log.Debugf("%s: join token ID: %s", p.leader, tokenID) defer func() { - if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", h.DataDir, tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil { + if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", h.K0sDataDir(), tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil { log.Warnf("%s: failed to invalidate the controller join token", p.leader) } }() diff --git a/phase/install_workers.go b/phase/install_workers.go index 89decf73..d9f88d03 100644 --- a/phase/install_workers.go +++ b/phase/install_workers.go @@ -85,7 +85,7 @@ func (p *InstallWorkers) Run() error { if !NoWait { defer func() { - if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", p.leader.DataDir, tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil { + if err := p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", p.leader.K0sDataDir(), tokenID), exec.Sudo(p.leader), exec.RedactString(token)); err != nil { log.Warnf("%s: failed to invalidate the worker join token", p.leader) } }() diff --git a/phase/prepare_hosts.go b/phase/prepare_hosts.go index 7d118906..d5697aca 100644 --- a/phase/prepare_hosts.go +++ b/phase/prepare_hosts.go @@ -3,7 +3,6 @@ package phase import ( "strings" - "github.com/alessio/shellescape" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" "github.com/k0sproject/rig/os" log "github.com/sirupsen/logrus" @@ -70,11 +69,5 @@ func (p *PrepareHosts) prepareHost(h *cluster.Host) error { } } - if h.DataDir == "" { - log.Debugf("%s: data-dir is not set, using default", h) - h.DataDir = h.Configurer.DataDirDefaultPath() - } - h.DataDir = shellescape.Quote(h.DataDir) - return nil } diff --git a/phase/reset_controllers.go b/phase/reset_controllers.go index b1f8f53c..166a518b 100644 --- a/phase/reset_controllers.go +++ b/phase/reset_controllers.go @@ -103,7 +103,7 @@ func (p *ResetControllers) Run() error { log.Debugf("%s: leaving etcd completed", h) log.Debugf("%s: resetting k0s...", h) - out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.DataDir), exec.Sudo(h)) + out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)) if err != nil { log.Debugf("%s: k0s reset failed: %s", h, out) log.Warnf("%s: k0s reported failure: %v", h, err) diff --git a/phase/reset_leader.go b/phase/reset_leader.go index e537fc02..1a4cf668 100644 --- a/phase/reset_leader.go +++ b/phase/reset_leader.go @@ -49,7 +49,7 @@ func (p *ResetLeader) Run() error { } log.Debugf("%s: resetting k0s...", p.leader) - out, err := p.leader.ExecOutput(p.leader.Configurer.K0sCmdf("reset --data-dir=%s", p.leader.DataDir), exec.Sudo(p.leader)) + out, err := p.leader.ExecOutput(p.leader.Configurer.K0sCmdf("reset --data-dir=%s", p.leader.K0sDataDir()), exec.Sudo(p.leader)) if err != nil { log.Debugf("%s: k0s reset failed: %s", p.leader, out) log.Warnf("%s: k0s reported failure: %v", p.leader, err) diff --git a/phase/reset_workers.go b/phase/reset_workers.go index dbd7aa37..e3ae829a 100644 --- a/phase/reset_workers.go +++ b/phase/reset_workers.go @@ -94,7 +94,7 @@ func (p *ResetWorkers) Run() error { } log.Debugf("%s: resetting k0s...", h) - out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.DataDir), exec.Sudo(h)) + out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset --data-dir=%s", h.K0sDataDir()), exec.Sudo(h)) if err != nil { log.Debugf("%s: k0s reset failed: %s", h, out) log.Warnf("%s: k0s reported failure: %v", h, err) diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go index e64f5859..3880cf70 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go @@ -125,8 +125,8 @@ type configurer interface { DeleteFile(os.Host, string) error CommandExist(os.Host, string) bool Hostname(os.Host) string - KubectlCmdf(os.Host, string, ...interface{}) string - KubeconfigPath(os.Host) string + KubectlCmdf(os.Host, string, string, ...interface{}) string + KubeconfigPath(os.Host, string) string IsContainer(os.Host) bool FixContainer(os.Host) error HTTPStatus(os.Host, string) (int, error) @@ -260,7 +260,7 @@ func (h *Host) K0sInstallCommand() (string, error) { role := h.Role flags := h.InstallFlags - flags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.DataDir)) + flags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.K0sDataDir())) switch role { case "controller+worker": @@ -330,12 +330,12 @@ func (h *Host) K0sInstallCommand() (string, error) { // K0sBackupCommand returns a full command to be used as run k0s backup func (h *Host) K0sBackupCommand(targetDir string) string { - return h.Configurer.K0sCmdf("backup --save-path %s --data-dir %s", shellescape.Quote(targetDir), h.DataDir) + return h.Configurer.K0sCmdf("backup --save-path %s --data-dir %s", shellescape.Quote(targetDir), h.K0sDataDir()) } // K0sRestoreCommand returns a full command to restore cluster state from a backup func (h *Host) K0sRestoreCommand(backupfile string) string { - return h.Configurer.K0sCmdf("restore --data-dir=%s %s", h.DataDir, shellescape.Quote(backupfile)) + return h.Configurer.K0sCmdf("restore --data-dir=%s %s", h.K0sDataDir(), shellescape.Quote(backupfile)) } // IsController returns true for controller and controller+worker roles @@ -390,6 +390,14 @@ func (h *Host) UpdateK0sBinary(path string, version *version.Version) error { return nil } +// K0sDataDir returns the data dir for the host either from host.DataDir or the default from configurer's DataDirDefaultPath +func (h *Host) K0sDataDir() string { + if h.DataDir == "" { + return h.Configurer.DataDirDefaultPath() + } + return h.DataDir +} + type kubeNodeStatus struct { Items []struct { Status struct { @@ -403,7 +411,7 @@ type kubeNodeStatus struct { // KubeNodeReady runs kubectl on the host and returns true if the given node is marked as ready func (h *Host) KubeNodeReady() (bool, error) { - output, err := h.ExecOutput(h.Configurer.KubectlCmdf(h, "get node --data-dir=%s -l kubernetes.io/hostname=%s -o json", h.DataDir, h.Metadata.Hostname), exec.HideOutput(), exec.Sudo(h)) + output, err := h.ExecOutput(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "get node -l kubernetes.io/hostname=%s -o json", h.Metadata.Hostname), exec.HideOutput(), exec.Sudo(h)) if err != nil { return false, err } @@ -456,7 +464,7 @@ type statusEvents struct { func (h *Host) WaitK0sDynamicConfigReady() error { return retry.Do( func() error { - output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s -o json", h.DataDir), exec.Sudo(h)) + output, err := h.ExecOutput(h.Configurer.K0sCmdf("kubectl --data-dir=%s -n kube-system get event --field-selector involvedObject.name=k0s -o json", h.K0sDataDir()), exec.Sudo(h)) if err != nil { return fmt.Errorf("failed to get k0s config status events: %w", err) } @@ -481,17 +489,17 @@ func (h *Host) WaitK0sDynamicConfigReady() error { // DrainNode drains the given node func (h *Host) DrainNode(node *Host) error { - return h.Exec(h.Configurer.KubectlCmdf(h, "drain --data-dir=%s --grace-period=120 --force --timeout=5m --ignore-daemonsets --delete-emptydir-data %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h)) + return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "drain --grace-period=120 --force --timeout=5m --ignore-daemonsets --delete-emptydir-data %s", node.Metadata.Hostname), exec.Sudo(h)) } // UncordonNode marks the node schedulable again func (h *Host) UncordonNode(node *Host) error { - return h.Exec(h.Configurer.KubectlCmdf(h, "uncordon --data-dir=%s %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h)) + return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "uncordon %s", node.Metadata.Hostname), exec.Sudo(h)) } // DeleteNode deletes the given node from kubernetes func (h *Host) DeleteNode(node *Host) error { - return h.Exec(h.Configurer.KubectlCmdf(h, "delete node --data-dir=%s %s", h.DataDir, node.Metadata.Hostname), exec.Sudo(h)) + return h.Exec(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "delete node %s", node.Metadata.Hostname), exec.Sudo(h)) } func (h *Host) LeaveEtcd(node *Host) error { @@ -499,7 +507,7 @@ func (h *Host) LeaveEtcd(node *Host) error { if node.PrivateAddress != "" { etcdAddress = node.PrivateAddress } - return h.Exec(h.Configurer.K0sCmdf("etcd leave --peer-address %s --datadir %s", etcdAddress, h.DataDir), exec.Sudo(h)) + return h.Exec(h.Configurer.K0sCmdf("etcd leave --peer-address %s --datadir %s", etcdAddress, h.K0sDataDir()), exec.Sudo(h)) } // CheckHTTPStatus will perform a web request to the url and return an error if the http status is not the expected diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go index 3e226e92..61a5e971 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go @@ -144,7 +144,7 @@ func (k K0s) GenerateToken(h *Host, role string, expiry time.Duration) (string, k0sFlags.Add(fmt.Sprintf("--config %s", shellescape.Quote(h.K0sConfigPath()))) } - k0sFlags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.DataDir)) + k0sFlags.AddOrReplace(fmt.Sprintf("--data-dir=%s", h.K0sDataDir())) var token string err = retry.Do( @@ -167,7 +167,7 @@ func (k K0s) GenerateToken(h *Host, role string, expiry time.Duration) (string, // GetClusterID uses kubectl to fetch the kube-system namespace uid func (k K0s) GetClusterID(h *Host) (string, error) { - return h.ExecOutput(h.Configurer.KubectlCmdf(h, "get --data-dir=%s -n kube-system namespace kube-system -o template={{.metadata.uid}}", h.DataDir), exec.Sudo(h)) + return h.ExecOutput(h.Configurer.KubectlCmdf(h, h.K0sDataDir(), "get -n kube-system namespace kube-system -o template={{.metadata.uid}}"), exec.Sudo(h)) } // VersionEqual returns true if the configured k0s version is equal to the given version string