From da77555539819419461a981e83a7aec8c648e3ad Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Tue, 19 Nov 2024 13:23:24 +0000 Subject: [PATCH 1/4] replace k8s-snap-api --- src/k8s/go.mod | 2 ++ src/k8s/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/k8s/go.mod b/src/k8s/go.mod index 68d3f1cde..8b9a8bcc7 100644 --- a/src/k8s/go.mod +++ b/src/k8s/go.mod @@ -177,3 +177,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/canonical/k8s-snap-api v1.0.13 => github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14 diff --git a/src/k8s/go.sum b/src/k8s/go.sum index 20f296a62..5b6cae72f 100644 --- a/src/k8s/go.sum +++ b/src/k8s/go.sum @@ -99,8 +99,6 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXe github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/canonical/go-dqlite v1.22.0 h1:DuJmfcREl4gkQJyvZzjl2GHFZROhbPyfdjDRQXpkOyw= github.com/canonical/go-dqlite v1.22.0/go.mod h1:Uvy943N8R4CFUAs59A1NVaziWY9nJ686lScY7ywurfg= -github.com/canonical/k8s-snap-api v1.0.13 h1:Z+IW6Knvycu+DrkmH+9qB1UNyYiHfL+rFvT9DtSO2+g= -github.com/canonical/k8s-snap-api v1.0.13/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230 h1:YOqZ+/14OPZ+/TOXpRHIX3KLT0C+wZVpewKIwlGUmW0= github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230/go.mod h1:YVGI7HStOKsV+cMyXWnJ7RaMPaeWtrkxyIPvGWbgACc= github.com/canonical/microcluster/v3 v3.0.0-20240827143335-f7a4d3984970 h1:UrnpglbXELlxtufdk6DGDytu2JzyzuS3WTsOwPrkQLI= @@ -113,6 +111,8 @@ github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHe github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14 h1:M5U5soMZ7Gn6NgSeE3eDtO/qX3UsUBq2u0+RODZsgXU= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= From 10f4d2e58cc7730bfcc778011e7f2766473756b9 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Tue, 19 Nov 2024 14:59:15 +0000 Subject: [PATCH 2/4] Adds configurable containerd base dir during bootstrap / join Currently, for the classic k8s snap, the default locations for the containerd-related files are: - /etc/containerd/ - /run/containerd/ - /var/lib/containerd/ These paths can conflict with other containerd installations on the host (e.g. from docker), meaning that the k8s snap cannot be installed. In a developer's usecase, we shouldn't require the other services to be disabled, but we can allow the k8s snap's containerd to be installed in a different location during bootstrap / node join. Adds the --containerd-base-dir option for k8s bootstrap and join-cluster commands. If not given, the snap's default paths will be used instead. After the node was initialized, save the containerd base location in a file, so we can properly reference it later / on restart. --- docs/src/_parts/commands/k8s_bootstrap.md | 15 +++++++------ docs/src/_parts/commands/k8s_join-cluster.md | 13 ++++++----- src/k8s/cmd/k8s/k8s_bootstrap.go | 23 +++++++++++--------- src/k8s/cmd/k8s/k8s_join_cluster.go | 23 +++++++++++--------- src/k8s/cmd/util/environ.go | 14 +++++++++--- src/k8s/pkg/k8sd/api/cluster_bootstrap.go | 5 +++++ src/k8s/pkg/k8sd/api/cluster_join.go | 5 +++++ src/k8s/pkg/k8sd/setup/containerd.go | 1 + src/k8s/pkg/snap/interface.go | 2 ++ src/k8s/pkg/snap/mock/mock.go | 9 ++++++++ src/k8s/pkg/snap/snap.go | 15 ++++++++++++- 11 files changed, 88 insertions(+), 37 deletions(-) diff --git a/docs/src/_parts/commands/k8s_bootstrap.md b/docs/src/_parts/commands/k8s_bootstrap.md index 1ad241588..93935315e 100644 --- a/docs/src/_parts/commands/k8s_bootstrap.md +++ b/docs/src/_parts/commands/k8s_bootstrap.md @@ -13,13 +13,14 @@ k8s bootstrap [flags] ### Options ``` - --address string microcluster address or CIDR, defaults to the node IP address - --file string path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin. - -h, --help help for bootstrap - --interactive interactively configure the most important cluster options - --name string node name, defaults to hostname - --output-format string set the output format to one of plain, json or yaml (default "plain") - --timeout duration the max time to wait for the command to execute (default 1m30s) + --address string microcluster address or CIDR, defaults to the node IP address + --containerd-base-dir string set the containerd base containerd directory + --file string path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin. + -h, --help help for bootstrap + --interactive interactively configure the most important cluster options + --name string node name, defaults to hostname + --output-format string set the output format to one of plain, json or yaml (default "plain") + --timeout duration the max time to wait for the command to execute (default 1m30s) ``` ### SEE ALSO diff --git a/docs/src/_parts/commands/k8s_join-cluster.md b/docs/src/_parts/commands/k8s_join-cluster.md index d485b902a..15be28ef3 100644 --- a/docs/src/_parts/commands/k8s_join-cluster.md +++ b/docs/src/_parts/commands/k8s_join-cluster.md @@ -9,12 +9,13 @@ k8s join-cluster [flags] ### Options ``` - --address string microcluster address or CIDR, defaults to the node IP address - --file string path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin. - -h, --help help for join-cluster - --name string node name, defaults to hostname - --output-format string set the output format to one of plain, json or yaml (default "plain") - --timeout duration the max time to wait for the command to execute (default 1m30s) + --address string microcluster address or CIDR, defaults to the node IP address + --containerd-base-dir string set the containerd base containerd directory + --file string path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin. + -h, --help help for join-cluster + --name string node name, defaults to hostname + --output-format string set the output format to one of plain, json or yaml (default "plain") + --timeout duration the max time to wait for the command to execute (default 1m30s) ``` ### SEE ALSO diff --git a/src/k8s/cmd/k8s/k8s_bootstrap.go b/src/k8s/cmd/k8s/k8s_bootstrap.go index 9ba6525d3..a982ecdeb 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap.go @@ -35,12 +35,13 @@ func (b BootstrapResult) String() string { func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { var opts struct { - interactive bool - configFile string - name string - address string - outputFormat string - timeout time.Duration + interactive bool + configFile string + name string + address string + outputFormat string + timeout time.Duration + containerdBaseDir string } cmd := &cobra.Command{ Use: "bootstrap", @@ -149,10 +150,11 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.PrintErrln("Bootstrapping the cluster. This may take a few seconds, please wait.") response, err := client.BootstrapCluster(cmd.Context(), apiv1.BootstrapClusterRequest{ - Name: opts.name, - Address: address, - Config: bootstrapConfig, - Timeout: opts.timeout, + Name: opts.name, + Address: address, + Config: bootstrapConfig, + Timeout: opts.timeout, + ContainerdBaseDir: opts.containerdBaseDir, }) if err != nil { cmd.PrintErrf("Error: Failed to bootstrap the cluster.\n\nThe error was: %v\n", err) @@ -170,6 +172,7 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.Flags().StringVar(&opts.address, "address", "", "microcluster address or CIDR, defaults to the node IP address") cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml") cmd.Flags().DurationVar(&opts.timeout, "timeout", 90*time.Second, "the max time to wait for the command to execute") + cmd.Flags().StringVar(&opts.containerdBaseDir, "containerd-base-dir", "", "set the containerd base containerd directory") return cmd } diff --git a/src/k8s/cmd/k8s/k8s_join_cluster.go b/src/k8s/cmd/k8s/k8s_join_cluster.go index 4cd5bfe6d..d2ecdf5d7 100644 --- a/src/k8s/cmd/k8s/k8s_join_cluster.go +++ b/src/k8s/cmd/k8s/k8s_join_cluster.go @@ -23,11 +23,12 @@ func (b JoinClusterResult) String() string { func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { var opts struct { - name string - address string - configFile string - outputFormat string - timeout time.Duration + name string + address string + configFile string + outputFormat string + timeout time.Duration + containerdBaseDir string } cmd := &cobra.Command{ Use: "join-cluster ", @@ -103,11 +104,12 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.PrintErrln("Joining the cluster. This may take a few seconds, please wait.") if err := client.JoinCluster(cmd.Context(), apiv1.JoinClusterRequest{ - Name: opts.name, - Address: address, - Token: token, - Config: joinClusterConfig, - Timeout: opts.timeout, + Name: opts.name, + Address: address, + Token: token, + Config: joinClusterConfig, + Timeout: opts.timeout, + ContainerdBaseDir: opts.containerdBaseDir, }); err != nil { cmd.PrintErrf("Error: Failed to join the cluster using the provided token.\n\nThe error was: %v\n", err) env.Exit(1) @@ -122,5 +124,6 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin.") cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml") cmd.Flags().DurationVar(&opts.timeout, "timeout", 90*time.Second, "the max time to wait for the command to execute") + cmd.Flags().StringVar(&opts.containerdBaseDir, "containerd-base-dir", "", "set the containerd base containerd directory") return cmd } diff --git a/src/k8s/cmd/util/environ.go b/src/k8s/cmd/util/environ.go index d843f71f0..161a223c8 100644 --- a/src/k8s/cmd/util/environ.go +++ b/src/k8s/cmd/util/environ.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/canonical/k8s/pkg/snap" @@ -32,10 +33,17 @@ func DefaultExecutionEnvironment() ExecutionEnvironment { var s snap.Snap switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") { case "", "snap": + // If this node is already bootstrapped / joined, we should already know where the + // containerd base directory is. If not, leave the defaults. + containerdBaseDir := "" + if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.CONTAINERD_BASE_DIR)); err == nil { + containerdBaseDir = strings.TrimSpace(string(data)) + } s = snap.NewSnap(snap.SnapOpts{ - SnapDir: os.Getenv("SNAP"), - SnapCommonDir: os.Getenv("SNAP_COMMON"), - SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"), + SnapDir: os.Getenv("SNAP"), + SnapCommonDir: os.Getenv("SNAP_COMMON"), + SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"), + ContainerdBaseDir: containerdBaseDir, }) case "pebble": s = snap.NewPebble(snap.PebbleOpts{ diff --git a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go index 3c0a7391b..3475fb44f 100644 --- a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go +++ b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go @@ -36,6 +36,11 @@ func (e *Endpoints) postClusterBootstrap(_ state.State, r *http.Request) respons return response.BadRequest(fmt.Errorf("cluster is already bootstrapped")) } + // If not set, leave the default base dir location. + if req.ContainerdBaseDir != "" { + e.provider.Snap().SetContainerdBaseDir(req.ContainerdBaseDir) + } + // NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set. ctx, cancel := context.WithTimeout(r.Context(), time.Hour) defer cancel() diff --git a/src/k8s/pkg/k8sd/api/cluster_join.go b/src/k8s/pkg/k8sd/api/cluster_join.go index 3d19997db..7ee9a61d6 100644 --- a/src/k8s/pkg/k8sd/api/cluster_join.go +++ b/src/k8s/pkg/k8sd/api/cluster_join.go @@ -28,6 +28,11 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res return NodeInUse(fmt.Errorf("node %q is part of the cluster", hostname)) } + // If not set, leave the default base dir location. + if req.ContainerdBaseDir != "" { + e.provider.Snap().SetContainerdBaseDir(req.ContainerdBaseDir) + } + config := map[string]string{} // NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set. diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index eb9909124..4dafdf2a8 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -173,6 +173,7 @@ func saveSnapContainerdPaths(s snap.Snap) error { "containerd-config-dir": s.ContainerdConfigDir(), "containerd-root-dir": s.ContainerdRootDir(), "containerd-cni-bin-dir": s.CNIBinDir(), + snap.CONTAINERD_BASE_DIR: s.GetContainerdBaseDir(), } for filename, content := range m { diff --git a/src/k8s/pkg/snap/interface.go b/src/k8s/pkg/snap/interface.go index f2ea533df..b26087f5c 100644 --- a/src/k8s/pkg/snap/interface.go +++ b/src/k8s/pkg/snap/interface.go @@ -39,6 +39,8 @@ type Snap interface { EtcdPKIDir() string // /etc/kubernetes/pki/etcd KubeletRootDir() string // /var/lib/kubelet + SetContainerdBaseDir(baseDir string) // sets the containerd base directory. + GetContainerdBaseDir() string // gets the containerd base directory. ContainerdConfigDir() string // classic confinement: /etc/containerd, strict confinement: /var/snap/k8s/common/etc/containerd ContainerdExtraConfigDir() string // classic confinement: /etc/containerd/conf.d, strict confinement: /var/snap/k8s/common/etc/containerd/conf.d ContainerdRegistryConfigDir() string // classic confinement: /etc/containerd/hosts.d, strict confinement: /var/snap/k8s/common/etc/containerd/hosts.d diff --git a/src/k8s/pkg/snap/mock/mock.go b/src/k8s/pkg/snap/mock/mock.go index 21846253d..544a65471 100644 --- a/src/k8s/pkg/snap/mock/mock.go +++ b/src/k8s/pkg/snap/mock/mock.go @@ -30,6 +30,7 @@ type Mock struct { ContainerdConfigDir string ContainerdExtraConfigDir string ContainerdRegistryConfigDir string + ContainerdBaseDir string ContainerdRootDir string ContainerdSocketDir string ContainerdSocketPath string @@ -131,6 +132,14 @@ func (s *Snap) Hostname() string { return s.Mock.Hostname } +func (s *Snap) SetContainerdBaseDir(baseDir string) { + s.Mock.ContainerdBaseDir = baseDir +} + +func (s *Snap) GetContainerdBaseDir() string { + return s.Mock.ContainerdBaseDir +} + func (s *Snap) ContainerdConfigDir() string { return s.Mock.ContainerdConfigDir } diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index 057d30cde..bc6b8d5ba 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -23,6 +23,10 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" ) +const ( + CONTAINERD_BASE_DIR = ".containerd-base-dir" +) + type SnapOpts struct { SnapInstanceName string SnapDir string @@ -172,6 +176,14 @@ func (s *snap) Hostname() string { return hostname } +func (s *snap) SetContainerdBaseDir(baseDir string) { + s.containerdBaseDir = baseDir +} + +func (s *snap) GetContainerdBaseDir() string { + return s.containerdBaseDir +} + func (s *snap) ContainerdConfigDir() string { return filepath.Join(s.containerdBaseDir, "etc", "containerd") } @@ -346,7 +358,8 @@ func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) er if _, err := os.Stat(s.ContainerdSocketDir()); err == nil { return fmt.Errorf("The path '%s' required for the containerd socket already exists. "+ "This may mean that another service is already using that path, and it conflicts with the k8s snap. "+ - "Please make sure that there is no other service installed that uses the same path, and remove the existing directory.", s.ContainerdSocketDir()) + "Please make sure that there is no other service installed that uses the same path, and remove the existing directory."+ + "(dev-only): You can change the default k8s containerd base path with the --containerd-base-dir flag when bootstrapping / joining.", s.ContainerdSocketDir()) } else if !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Encountered an error while checking '%s': %w", s.ContainerdSocketDir(), err) } From 56f7b080da8a5b61c9c108e1b282f1a8a2a41d43 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Thu, 21 Nov 2024 10:18:14 +0000 Subject: [PATCH 3/4] Changes ContainerdBaseDir to a bootstrap config option --- docs/src/_parts/bootstrap_config.md | 4 ++++ docs/src/_parts/commands/k8s_bootstrap.md | 15 ++++++------- docs/src/_parts/commands/k8s_join-cluster.md | 13 +++++------ docs/src/_parts/control_plane_join_config.md | 4 ++++ docs/src/_parts/worker_join_config.md | 4 ++++ k8s/lib.sh | 1 + src/k8s/cmd/k8s/k8s_bootstrap.go | 23 +++++++++----------- src/k8s/cmd/k8s/k8s_join_cluster.go | 23 +++++++++----------- src/k8s/cmd/util/environ.go | 2 +- src/k8s/go.mod | 2 +- src/k8s/go.sum | 4 ++-- src/k8s/pkg/k8sd/api/cluster_bootstrap.go | 6 +++-- src/k8s/pkg/k8sd/api/cluster_join.go | 17 ++++++++++++--- src/k8s/pkg/k8sd/setup/containerd.go | 2 +- src/k8s/pkg/snap/snap.go | 4 ++-- 15 files changed, 71 insertions(+), 53 deletions(-) diff --git a/docs/src/_parts/bootstrap_config.md b/docs/src/_parts/bootstrap_config.md index b1ed6f477..a719a138e 100644 --- a/docs/src/_parts/bootstrap_config.md +++ b/docs/src/_parts/bootstrap_config.md @@ -483,3 +483,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/docs/src/_parts/commands/k8s_bootstrap.md b/docs/src/_parts/commands/k8s_bootstrap.md index 93935315e..1ad241588 100644 --- a/docs/src/_parts/commands/k8s_bootstrap.md +++ b/docs/src/_parts/commands/k8s_bootstrap.md @@ -13,14 +13,13 @@ k8s bootstrap [flags] ### Options ``` - --address string microcluster address or CIDR, defaults to the node IP address - --containerd-base-dir string set the containerd base containerd directory - --file string path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin. - -h, --help help for bootstrap - --interactive interactively configure the most important cluster options - --name string node name, defaults to hostname - --output-format string set the output format to one of plain, json or yaml (default "plain") - --timeout duration the max time to wait for the command to execute (default 1m30s) + --address string microcluster address or CIDR, defaults to the node IP address + --file string path to the YAML file containing your custom cluster bootstrap configuration. Use '-' to read from stdin. + -h, --help help for bootstrap + --interactive interactively configure the most important cluster options + --name string node name, defaults to hostname + --output-format string set the output format to one of plain, json or yaml (default "plain") + --timeout duration the max time to wait for the command to execute (default 1m30s) ``` ### SEE ALSO diff --git a/docs/src/_parts/commands/k8s_join-cluster.md b/docs/src/_parts/commands/k8s_join-cluster.md index 15be28ef3..d485b902a 100644 --- a/docs/src/_parts/commands/k8s_join-cluster.md +++ b/docs/src/_parts/commands/k8s_join-cluster.md @@ -9,13 +9,12 @@ k8s join-cluster [flags] ### Options ``` - --address string microcluster address or CIDR, defaults to the node IP address - --containerd-base-dir string set the containerd base containerd directory - --file string path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin. - -h, --help help for join-cluster - --name string node name, defaults to hostname - --output-format string set the output format to one of plain, json or yaml (default "plain") - --timeout duration the max time to wait for the command to execute (default 1m30s) + --address string microcluster address or CIDR, defaults to the node IP address + --file string path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin. + -h, --help help for join-cluster + --name string node name, defaults to hostname + --output-format string set the output format to one of plain, json or yaml (default "plain") + --timeout duration the max time to wait for the command to execute (default 1m30s) ``` ### SEE ALSO diff --git a/docs/src/_parts/control_plane_join_config.md b/docs/src/_parts/control_plane_join_config.md index fa2919e45..6bda426af 100644 --- a/docs/src/_parts/control_plane_join_config.md +++ b/docs/src/_parts/control_plane_join_config.md @@ -150,3 +150,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/docs/src/_parts/worker_join_config.md b/docs/src/_parts/worker_join_config.md index 70a515a8f..58e1798c1 100644 --- a/docs/src/_parts/worker_join_config.md +++ b/docs/src/_parts/worker_join_config.md @@ -76,3 +76,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/k8s/lib.sh b/k8s/lib.sh index e81b7aefd..032878795 100755 --- a/k8s/lib.sh +++ b/k8s/lib.sh @@ -99,6 +99,7 @@ k8s::remove::containerd() { # only remove containerd if the snap was already bootstrapped. # this is to prevent removing containerd when it is not installed by the snap. + # NOTE: do NOT include .containerd-base-dir! for file in "containerd-socket-path" "containerd-config-dir" "containerd-root-dir" "containerd-cni-bin-dir"; do if [ -f "$SNAP_COMMON/lock/$file" ]; then rm -rf $(cat "$SNAP_COMMON/lock/$file") diff --git a/src/k8s/cmd/k8s/k8s_bootstrap.go b/src/k8s/cmd/k8s/k8s_bootstrap.go index a982ecdeb..9ba6525d3 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap.go @@ -35,13 +35,12 @@ func (b BootstrapResult) String() string { func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { var opts struct { - interactive bool - configFile string - name string - address string - outputFormat string - timeout time.Duration - containerdBaseDir string + interactive bool + configFile string + name string + address string + outputFormat string + timeout time.Duration } cmd := &cobra.Command{ Use: "bootstrap", @@ -150,11 +149,10 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.PrintErrln("Bootstrapping the cluster. This may take a few seconds, please wait.") response, err := client.BootstrapCluster(cmd.Context(), apiv1.BootstrapClusterRequest{ - Name: opts.name, - Address: address, - Config: bootstrapConfig, - Timeout: opts.timeout, - ContainerdBaseDir: opts.containerdBaseDir, + Name: opts.name, + Address: address, + Config: bootstrapConfig, + Timeout: opts.timeout, }) if err != nil { cmd.PrintErrf("Error: Failed to bootstrap the cluster.\n\nThe error was: %v\n", err) @@ -172,7 +170,6 @@ func newBootstrapCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.Flags().StringVar(&opts.address, "address", "", "microcluster address or CIDR, defaults to the node IP address") cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml") cmd.Flags().DurationVar(&opts.timeout, "timeout", 90*time.Second, "the max time to wait for the command to execute") - cmd.Flags().StringVar(&opts.containerdBaseDir, "containerd-base-dir", "", "set the containerd base containerd directory") return cmd } diff --git a/src/k8s/cmd/k8s/k8s_join_cluster.go b/src/k8s/cmd/k8s/k8s_join_cluster.go index d2ecdf5d7..4cd5bfe6d 100644 --- a/src/k8s/cmd/k8s/k8s_join_cluster.go +++ b/src/k8s/cmd/k8s/k8s_join_cluster.go @@ -23,12 +23,11 @@ func (b JoinClusterResult) String() string { func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { var opts struct { - name string - address string - configFile string - outputFormat string - timeout time.Duration - containerdBaseDir string + name string + address string + configFile string + outputFormat string + timeout time.Duration } cmd := &cobra.Command{ Use: "join-cluster ", @@ -104,12 +103,11 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.PrintErrln("Joining the cluster. This may take a few seconds, please wait.") if err := client.JoinCluster(cmd.Context(), apiv1.JoinClusterRequest{ - Name: opts.name, - Address: address, - Token: token, - Config: joinClusterConfig, - Timeout: opts.timeout, - ContainerdBaseDir: opts.containerdBaseDir, + Name: opts.name, + Address: address, + Token: token, + Config: joinClusterConfig, + Timeout: opts.timeout, }); err != nil { cmd.PrintErrf("Error: Failed to join the cluster using the provided token.\n\nThe error was: %v\n", err) env.Exit(1) @@ -124,6 +122,5 @@ func newJoinClusterCmd(env cmdutil.ExecutionEnvironment) *cobra.Command { cmd.Flags().StringVar(&opts.configFile, "file", "", "path to the YAML file containing your custom cluster join configuration. Use '-' to read from stdin.") cmd.Flags().StringVar(&opts.outputFormat, "output-format", "plain", "set the output format to one of plain, json or yaml") cmd.Flags().DurationVar(&opts.timeout, "timeout", 90*time.Second, "the max time to wait for the command to execute") - cmd.Flags().StringVar(&opts.containerdBaseDir, "containerd-base-dir", "", "set the containerd base containerd directory") return cmd } diff --git a/src/k8s/cmd/util/environ.go b/src/k8s/cmd/util/environ.go index 161a223c8..2d3620ed1 100644 --- a/src/k8s/cmd/util/environ.go +++ b/src/k8s/cmd/util/environ.go @@ -36,7 +36,7 @@ func DefaultExecutionEnvironment() ExecutionEnvironment { // If this node is already bootstrapped / joined, we should already know where the // containerd base directory is. If not, leave the defaults. containerdBaseDir := "" - if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.CONTAINERD_BASE_DIR)); err == nil { + if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.ContainerdBaseDir)); err == nil { containerdBaseDir = strings.TrimSpace(string(data)) } s = snap.NewSnap(snap.SnapOpts{ diff --git a/src/k8s/go.mod b/src/k8s/go.mod index 8b9a8bcc7..5b00d08a8 100644 --- a/src/k8s/go.mod +++ b/src/k8s/go.mod @@ -178,4 +178,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/canonical/k8s-snap-api v1.0.13 => github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14 +replace github.com/canonical/k8s-snap-api v1.0.13 => github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153 diff --git a/src/k8s/go.sum b/src/k8s/go.sum index 5b6cae72f..4ecd1f1d5 100644 --- a/src/k8s/go.sum +++ b/src/k8s/go.sum @@ -111,8 +111,8 @@ github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHe github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14 h1:M5U5soMZ7Gn6NgSeE3eDtO/qX3UsUBq2u0+RODZsgXU= -github.com/claudiubelu/k8s-snap-api v0.0.0-20241119131040-dc75989e4d14/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153 h1:rSj/fRpUqEkwAJDJaWkuAUmGSPeKwP3a0sA7dCHVGvE= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= diff --git a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go index 3475fb44f..ede8f0473 100644 --- a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go +++ b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path/filepath" "time" apiv1 "github.com/canonical/k8s-snap-api/api/v1" @@ -37,8 +38,9 @@ func (e *Endpoints) postClusterBootstrap(_ state.State, r *http.Request) respons } // If not set, leave the default base dir location. - if req.ContainerdBaseDir != "" { - e.provider.Snap().SetContainerdBaseDir(req.ContainerdBaseDir) + if req.Config.ContainerdBaseDir != "" { + // append k8s-containerd to the given base dir, so we don't flood it with our own folders. + e.provider.Snap().SetContainerdBaseDir(filepath.Join(req.Config.ContainerdBaseDir, "k8s-containerd")) } // NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set. diff --git a/src/k8s/pkg/k8sd/api/cluster_join.go b/src/k8s/pkg/k8sd/api/cluster_join.go index 7ee9a61d6..9f43625b6 100644 --- a/src/k8s/pkg/k8sd/api/cluster_join.go +++ b/src/k8s/pkg/k8sd/api/cluster_join.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path/filepath" "time" apiv1 "github.com/canonical/k8s-snap-api/api/v1" @@ -11,6 +12,7 @@ import ( "github.com/canonical/k8s/pkg/utils" "github.com/canonical/lxd/lxd/response" "github.com/canonical/microcluster/v3/state" + "gopkg.in/yaml.v2" ) func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Response { @@ -28,9 +30,18 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res return NodeInUse(fmt.Errorf("node %q is part of the cluster", hostname)) } - // If not set, leave the default base dir location. - if req.ContainerdBaseDir != "" { - e.provider.Snap().SetContainerdBaseDir(req.ContainerdBaseDir) + joinConfig := struct { + // We only care about this field from the entire join config. + containerdBaseDir string `yaml:"containerd-base-dir,omitempty"` + }{} + + if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil { + return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err)) + } + + if joinConfig.containerdBaseDir != "" { + // append k8s-containerd to the given base dir, so we don't flood it with our own folders. + e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd")) } config := map[string]string{} diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index 4dafdf2a8..d6c6c8a10 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -173,7 +173,7 @@ func saveSnapContainerdPaths(s snap.Snap) error { "containerd-config-dir": s.ContainerdConfigDir(), "containerd-root-dir": s.ContainerdRootDir(), "containerd-cni-bin-dir": s.CNIBinDir(), - snap.CONTAINERD_BASE_DIR: s.GetContainerdBaseDir(), + snap.ContainerdBaseDir: s.GetContainerdBaseDir(), } for filename, content := range m { diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index bc6b8d5ba..f5d595379 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -24,7 +24,7 @@ import ( ) const ( - CONTAINERD_BASE_DIR = ".containerd-base-dir" + ContainerdBaseDir = ".containerd-base-dir" ) type SnapOpts struct { @@ -359,7 +359,7 @@ func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) er return fmt.Errorf("The path '%s' required for the containerd socket already exists. "+ "This may mean that another service is already using that path, and it conflicts with the k8s snap. "+ "Please make sure that there is no other service installed that uses the same path, and remove the existing directory."+ - "(dev-only): You can change the default k8s containerd base path with the --containerd-base-dir flag when bootstrapping / joining.", s.ContainerdSocketDir()) + "(dev-only): You can change the default k8s containerd base path with the containerd-base-dir option in the bootstrap / join-cluster config file.", s.ContainerdSocketDir()) } else if !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Encountered an error while checking '%s': %w", s.ContainerdSocketDir(), err) } From a862186d5cd2f4d109275039749c1b81da036319 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Fri, 22 Nov 2024 06:55:49 +0000 Subject: [PATCH 4/4] Adds node cleanup integration test for new containerd path --- .../templates/bootstrap-containerd-path.yaml | 2 + tests/integration/tests/test_cleanup.py | 63 +++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 tests/integration/templates/bootstrap-containerd-path.yaml diff --git a/tests/integration/templates/bootstrap-containerd-path.yaml b/tests/integration/templates/bootstrap-containerd-path.yaml new file mode 100644 index 000000000..5abfa7878 --- /dev/null +++ b/tests/integration/templates/bootstrap-containerd-path.yaml @@ -0,0 +1,2 @@ +# Contains the bootstrap configuration for the containerd-related paths test. +containerd-base-dir: /home/ubuntu diff --git a/tests/integration/tests/test_cleanup.py b/tests/integration/tests/test_cleanup.py index 784ac6b4a..1540ef5c8 100644 --- a/tests/integration/tests/test_cleanup.py +++ b/tests/integration/tests/test_cleanup.py @@ -2,19 +2,21 @@ # Copyright 2024 Canonical, Ltd. # import logging +import os from typing import List import pytest -from test_util import harness, util +import yaml +from test_util import config, harness, util LOG = logging.getLogger(__name__) CONTAINERD_PATHS = [ "/etc/containerd", - "/opt/cni/bin", "/run/containerd", "/var/lib/containerd", ] +CNI_PATH = "/opt/cni/bin" @pytest.mark.node_count(1) @@ -26,8 +28,61 @@ def test_node_cleanup(instances: List[harness.Instance]): util.remove_k8s_snap(instance) # Check that the containerd-related folders are removed on snap removal. + all_paths = CONTAINERD_PATHS + [CNI_PATH] process = instance.exec( - ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False + ["ls", CNI_PATH, *all_paths], capture_output=True, text=True, check=False ) - for path in CONTAINERD_PATHS: + for path in all_paths: assert f"cannot access '{path}': No such file or directory" in process.stderr + + +@pytest.mark.node_count(2) +@pytest.mark.disable_k8s_bootstrapping() +def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]): + main = instances[0] + joiner = instances[1] + + containerd_path_bootstrap_config = ( + config.MANIFESTS_DIR / "bootstrap-containerd-path.yaml" + ).read_text() + + main.exec( + ["k8s", "bootstrap", "--file", "-"], + input=str.encode(containerd_path_bootstrap_config), + ) + + join_token = util.get_join_token(main, joiner) + joiner.exec( + ["k8s", "join-cluster", join_token, "--file", "-"], + input=str.encode(containerd_path_bootstrap_config), + ) + + boostrap_config = yaml.safe_load(containerd_path_bootstrap_config) + new_containerd_paths = [ + os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p) + for p in CONTAINERD_PATHS + ] + for instance in instances: + # Check that the containerd-related folders are not in the default locations. + process = instance.exec( + ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False + ) + for path in CONTAINERD_PATHS: + assert ( + f"cannot access '{path}': No such file or directory" in process.stderr + ) + + # Check that the containerd-related folders are in the new locations. + # If one of them is missing, this should have a non-zero exit code. + instance.exec(["ls", *new_containerd_paths], check=True) + + for instance in instances: + # Check that the containerd-related folders are not in the new locations after snap removal. + util.remove_k8s_snap(instance) + process = instance.exec( + ["ls", *new_containerd_paths], capture_output=True, text=True, check=False + ) + for path in new_containerd_paths: + assert ( + f"cannot access '{path}': No such file or directory" in process.stderr + )