Adds configurable containerd base dir during bootstrap / join

docs/src/_parts/bootstrap_config.md

@@ -483,3 +483,7 @@ The format is `map[<--flag-name>]<value>`.
 
 Extra configuration for the containerd config.toml
 
+### containerd-base-dir
+**Type:** `string`<br>
+
+

docs/src/_parts/control_plane_join_config.md

@@ -150,3 +150,7 @@ The format is `map[<--flag-name>]<value>`.
 
 Extra configuration for the containerd config.toml
 
+### containerd-base-dir
+**Type:** `string`<br>
+
+

docs/src/_parts/worker_join_config.md

@@ -76,3 +76,7 @@ The format is `map[<--flag-name>]<value>`.
 
 Extra configuration for the containerd config.toml
 
+### containerd-base-dir
+**Type:** `string`<br>
+
+

k8s/lib.sh

@@ -99,6 +99,7 @@ k8s::remove::containerd() {
 
   # only remove containerd if the snap was already bootstrapped.
   # this is to prevent removing containerd when it is not installed by the snap.
+  # NOTE: do NOT include .containerd-base-dir!
   for file in "containerd-socket-path" "containerd-config-dir" "containerd-root-dir" "containerd-cni-bin-dir"; do
     if [ -f "$SNAP_COMMON/lock/$file" ]; then
       rm -rf $(cat "$SNAP_COMMON/lock/$file")

src/k8s/cmd/util/environ.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/canonical/k8s/pkg/snap"
@@ -32,10 +33,17 @@ func DefaultExecutionEnvironment() ExecutionEnvironment {
 	var s snap.Snap
 	switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") {
 	case "", "snap":
+		// If this node is already bootstrapped / joined, we should already know where the
+		// containerd base directory is. If not, leave the defaults.
+		containerdBaseDir := ""
+		if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.ContainerdBaseDir)); err == nil {
+			containerdBaseDir = strings.TrimSpace(string(data))
+		}
 		s = snap.NewSnap(snap.SnapOpts{
-			SnapDir:          os.Getenv("SNAP"),
-			SnapCommonDir:    os.Getenv("SNAP_COMMON"),
-			SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"),
+			SnapDir:           os.Getenv("SNAP"),
+			SnapCommonDir:     os.Getenv("SNAP_COMMON"),
+			SnapInstanceName:  os.Getenv("SNAP_INSTANCE_NAME"),
+			ContainerdBaseDir: containerdBaseDir,
 		})
 	case "pebble":
 		s = snap.NewPebble(snap.PebbleOpts{

src/k8s/go.mod

@@ -177,3 +177,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/canonical/k8s-snap-api v1.0.13 => github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153

src/k8s/go.sum

@@ -99,8 +99,6 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXe
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/canonical/go-dqlite v1.22.0 h1:DuJmfcREl4gkQJyvZzjl2GHFZROhbPyfdjDRQXpkOyw=
 github.com/canonical/go-dqlite v1.22.0/go.mod h1:Uvy943N8R4CFUAs59A1NVaziWY9nJ686lScY7ywurfg=
-github.com/canonical/k8s-snap-api v1.0.13 h1:Z+IW6Knvycu+DrkmH+9qB1UNyYiHfL+rFvT9DtSO2+g=
-github.com/canonical/k8s-snap-api v1.0.13/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY=
 github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230 h1:YOqZ+/14OPZ+/TOXpRHIX3KLT0C+wZVpewKIwlGUmW0=
 github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230/go.mod h1:YVGI7HStOKsV+cMyXWnJ7RaMPaeWtrkxyIPvGWbgACc=
 github.com/canonical/microcluster/v3 v3.0.0-20240827143335-f7a4d3984970 h1:UrnpglbXELlxtufdk6DGDytu2JzyzuS3WTsOwPrkQLI=
@@ -113,6 +111,8 @@ github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHe
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153 h1:rSj/fRpUqEkwAJDJaWkuAUmGSPeKwP3a0sA7dCHVGvE=
+github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=

src/k8s/pkg/k8sd/api/cluster_bootstrap.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	apiv1 "github.com/canonical/k8s-snap-api/api/v1"
@@ -36,6 +37,12 @@ func (e *Endpoints) postClusterBootstrap(_ state.State, r *http.Request) respons
 		return response.BadRequest(fmt.Errorf("cluster is already bootstrapped"))
 	}
 
+	// If not set, leave the default base dir location.
+	if req.Config.ContainerdBaseDir != "" {
+		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(req.Config.ContainerdBaseDir, "k8s-containerd"))
+	}
+
 	// NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set.
 	ctx, cancel := context.WithTimeout(r.Context(), time.Hour)
 	defer cancel()

src/k8s/pkg/k8sd/api/cluster_join.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	apiv1 "github.com/canonical/k8s-snap-api/api/v1"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/v3/state"
+	"gopkg.in/yaml.v2"
 )
 
 func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Response {
@@ -28,6 +30,20 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res
 		return NodeInUse(fmt.Errorf("node %q is part of the cluster", hostname))
 	}
 
+	joinConfig := struct {
+		// We only care about this field from the entire join config.
+		containerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
+	}{}
+
+	if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil {
+		return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err))
+	}
+
+	if joinConfig.containerdBaseDir != "" {
+		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd"))
+	}
+
 	config := map[string]string{}
 
 	// NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set.

src/k8s/pkg/k8sd/setup/containerd.go

@@ -173,6 +173,7 @@ func saveSnapContainerdPaths(s snap.Snap) error {
 		"containerd-config-dir":  s.ContainerdConfigDir(),
 		"containerd-root-dir":    s.ContainerdRootDir(),
 		"containerd-cni-bin-dir": s.CNIBinDir(),
+		snap.ContainerdBaseDir:   s.GetContainerdBaseDir(),
 	}
 
 	for filename, content := range m {

src/k8s/pkg/snap/interface.go

@@ -39,6 +39,8 @@ type Snap interface {
 	EtcdPKIDir() string          // /etc/kubernetes/pki/etcd
 	KubeletRootDir() string      // /var/lib/kubelet
 
+	SetContainerdBaseDir(baseDir string) // sets the containerd base directory.
+	GetContainerdBaseDir() string        // gets the containerd base directory.
 	ContainerdConfigDir() string         // classic confinement: /etc/containerd, strict confinement: /var/snap/k8s/common/etc/containerd
 	ContainerdExtraConfigDir() string    // classic confinement: /etc/containerd/conf.d, strict confinement: /var/snap/k8s/common/etc/containerd/conf.d
 	ContainerdRegistryConfigDir() string // classic confinement: /etc/containerd/hosts.d, strict confinement: /var/snap/k8s/common/etc/containerd/hosts.d

src/k8s/pkg/snap/mock/mock.go

@@ -30,6 +30,7 @@ type Mock struct {
 	ContainerdConfigDir         string
 	ContainerdExtraConfigDir    string
 	ContainerdRegistryConfigDir string
+	ContainerdBaseDir           string
 	ContainerdRootDir           string
 	ContainerdSocketDir         string
 	ContainerdSocketPath        string
@@ -131,6 +132,14 @@ func (s *Snap) Hostname() string {
 	return s.Mock.Hostname
 }
 
+func (s *Snap) SetContainerdBaseDir(baseDir string) {
+	s.Mock.ContainerdBaseDir = baseDir
+}
+
+func (s *Snap) GetContainerdBaseDir() string {
+	return s.Mock.ContainerdBaseDir
+}
+
 func (s *Snap) ContainerdConfigDir() string {
 	return s.Mock.ContainerdConfigDir
 }

src/k8s/pkg/snap/snap.go

@@ -23,6 +23,10 @@ import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
 
+const (
+	ContainerdBaseDir = ".containerd-base-dir"
+)
+
 type SnapOpts struct {
 	SnapInstanceName  string
 	SnapDir           string
@@ -172,6 +176,14 @@ func (s *snap) Hostname() string {
 	return hostname
 }
 
+func (s *snap) SetContainerdBaseDir(baseDir string) {
+	s.containerdBaseDir = baseDir
+}
+
+func (s *snap) GetContainerdBaseDir() string {
+	return s.containerdBaseDir
+}
+
 func (s *snap) ContainerdConfigDir() string {
 	return filepath.Join(s.containerdBaseDir, "etc", "containerd")
 }
@@ -346,7 +358,8 @@ func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) er
 	if _, err := os.Stat(s.ContainerdSocketDir()); err == nil {
 		return fmt.Errorf("The path '%s' required for the containerd socket already exists. "+
 			"This may mean that another service is already using that path, and it conflicts with the k8s snap. "+
-			"Please make sure that there is no other service installed that uses the same path, and remove the existing directory.", s.ContainerdSocketDir())
+			"Please make sure that there is no other service installed that uses the same path, and remove the existing directory."+
+			"(dev-only): You can change the default k8s containerd base path with the containerd-base-dir option in the bootstrap / join-cluster config file.", s.ContainerdSocketDir())
 	} else if !errors.Is(err, os.ErrNotExist) {
 		return fmt.Errorf("Encountered an error while checking '%s': %w", s.ContainerdSocketDir(), err)
 	}

tests/integration/templates/bootstrap-containerd-path.yaml

@@ -0,0 +1,2 @@
+# Contains the bootstrap configuration for the containerd-related paths test.
+containerd-base-dir: /home/ubuntu

tests/integration/tests/test_cleanup.py

@@ -2,19 +2,21 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import logging
+import os
 from typing import List
 
 import pytest
-from test_util import harness, util
+import yaml
+from test_util import config, harness, util
 
 LOG = logging.getLogger(__name__)
 
 CONTAINERD_PATHS = [
     "/etc/containerd",
-    "/opt/cni/bin",
     "/run/containerd",
     "/var/lib/containerd",
 ]
+CNI_PATH = "/opt/cni/bin"
 
 
 @pytest.mark.node_count(1)
@@ -26,8 +28,61 @@ def test_node_cleanup(instances: List[harness.Instance]):
     util.remove_k8s_snap(instance)
 
     # Check that the containerd-related folders are removed on snap removal.
+    all_paths = CONTAINERD_PATHS + [CNI_PATH]
     process = instance.exec(
-        ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+        ["ls", CNI_PATH, *all_paths], capture_output=True, text=True, check=False
     )
-    for path in CONTAINERD_PATHS:
+    for path in all_paths:
         assert f"cannot access '{path}': No such file or directory" in process.stderr
+
+
+@pytest.mark.node_count(2)
+@pytest.mark.disable_k8s_bootstrapping()
+def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
+    main = instances[0]
+    joiner = instances[1]
+
+    containerd_path_bootstrap_config = (
+        config.MANIFESTS_DIR / "bootstrap-containerd-path.yaml"
+    ).read_text()
+
+    main.exec(
+        ["k8s", "bootstrap", "--file", "-"],
+        input=str.encode(containerd_path_bootstrap_config),
+    )
+
+    join_token = util.get_join_token(main, joiner)
+    joiner.exec(
+        ["k8s", "join-cluster", join_token, "--file", "-"],
+        input=str.encode(containerd_path_bootstrap_config),
+    )
+
+    boostrap_config = yaml.safe_load(containerd_path_bootstrap_config)
+    new_containerd_paths = [
+        os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p)
+        for p in CONTAINERD_PATHS
+    ]
+    for instance in instances:
+        # Check that the containerd-related folders are not in the default locations.
+        process = instance.exec(
+            ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+        )
+        for path in CONTAINERD_PATHS:
+            assert (
+                f"cannot access '{path}': No such file or directory" in process.stderr
+            )
+
+        # Check that the containerd-related folders are in the new locations.
+        # If one of them is missing, this should have a non-zero exit code.
+        instance.exec(["ls", *new_containerd_paths], check=True)
+
+    for instance in instances:
+        # Check that the containerd-related folders are not in the new locations after snap removal.
+        util.remove_k8s_snap(instance)
+        process = instance.exec(
+            ["ls", *new_containerd_paths], capture_output=True, text=True, check=False
+        )
+        for path in new_containerd_paths:
+            assert (
+                f"cannot access '{path}': No such file or directory" in process.stderr
+            )