diff --git a/docs/src/_parts/bootstrap_config.md b/docs/src/_parts/bootstrap_config.md index b1ed6f477..a719a138e 100644 --- a/docs/src/_parts/bootstrap_config.md +++ b/docs/src/_parts/bootstrap_config.md @@ -483,3 +483,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/docs/src/_parts/control_plane_join_config.md b/docs/src/_parts/control_plane_join_config.md index fa2919e45..6bda426af 100644 --- a/docs/src/_parts/control_plane_join_config.md +++ b/docs/src/_parts/control_plane_join_config.md @@ -150,3 +150,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/docs/src/_parts/worker_join_config.md b/docs/src/_parts/worker_join_config.md index 70a515a8f..58e1798c1 100644 --- a/docs/src/_parts/worker_join_config.md +++ b/docs/src/_parts/worker_join_config.md @@ -76,3 +76,7 @@ The format is `map[<--flag-name>]`. Extra configuration for the containerd config.toml +### containerd-base-dir +**Type:** `string`
+ + diff --git a/k8s/lib.sh b/k8s/lib.sh index e81b7aefd..032878795 100755 --- a/k8s/lib.sh +++ b/k8s/lib.sh @@ -99,6 +99,7 @@ k8s::remove::containerd() { # only remove containerd if the snap was already bootstrapped. # this is to prevent removing containerd when it is not installed by the snap. + # NOTE: do NOT include .containerd-base-dir! for file in "containerd-socket-path" "containerd-config-dir" "containerd-root-dir" "containerd-cni-bin-dir"; do if [ -f "$SNAP_COMMON/lock/$file" ]; then rm -rf $(cat "$SNAP_COMMON/lock/$file") diff --git a/src/k8s/cmd/util/environ.go b/src/k8s/cmd/util/environ.go index d843f71f0..2d3620ed1 100644 --- a/src/k8s/cmd/util/environ.go +++ b/src/k8s/cmd/util/environ.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/canonical/k8s/pkg/snap" @@ -32,10 +33,17 @@ func DefaultExecutionEnvironment() ExecutionEnvironment { var s snap.Snap switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") { case "", "snap": + // If this node is already bootstrapped / joined, we should already know where the + // containerd base directory is. If not, leave the defaults. + containerdBaseDir := "" + if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.ContainerdBaseDir)); err == nil { + containerdBaseDir = strings.TrimSpace(string(data)) + } s = snap.NewSnap(snap.SnapOpts{ - SnapDir: os.Getenv("SNAP"), - SnapCommonDir: os.Getenv("SNAP_COMMON"), - SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"), + SnapDir: os.Getenv("SNAP"), + SnapCommonDir: os.Getenv("SNAP_COMMON"), + SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"), + ContainerdBaseDir: containerdBaseDir, }) case "pebble": s = snap.NewPebble(snap.PebbleOpts{ diff --git a/src/k8s/go.mod b/src/k8s/go.mod index 68d3f1cde..5b00d08a8 100644 --- a/src/k8s/go.mod +++ b/src/k8s/go.mod @@ -177,3 +177,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/canonical/k8s-snap-api v1.0.13 => github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153 diff --git a/src/k8s/go.sum b/src/k8s/go.sum index 20f296a62..4ecd1f1d5 100644 --- a/src/k8s/go.sum +++ b/src/k8s/go.sum @@ -99,8 +99,6 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXe github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/canonical/go-dqlite v1.22.0 h1:DuJmfcREl4gkQJyvZzjl2GHFZROhbPyfdjDRQXpkOyw= github.com/canonical/go-dqlite v1.22.0/go.mod h1:Uvy943N8R4CFUAs59A1NVaziWY9nJ686lScY7ywurfg= -github.com/canonical/k8s-snap-api v1.0.13 h1:Z+IW6Knvycu+DrkmH+9qB1UNyYiHfL+rFvT9DtSO2+g= -github.com/canonical/k8s-snap-api v1.0.13/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230 h1:YOqZ+/14OPZ+/TOXpRHIX3KLT0C+wZVpewKIwlGUmW0= github.com/canonical/lxd v0.0.0-20240822122218-e7b2a7a83230/go.mod h1:YVGI7HStOKsV+cMyXWnJ7RaMPaeWtrkxyIPvGWbgACc= github.com/canonical/microcluster/v3 v3.0.0-20240827143335-f7a4d3984970 h1:UrnpglbXELlxtufdk6DGDytu2JzyzuS3WTsOwPrkQLI= @@ -113,6 +111,8 @@ github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHe github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153 h1:rSj/fRpUqEkwAJDJaWkuAUmGSPeKwP3a0sA7dCHVGvE= +github.com/claudiubelu/k8s-snap-api v0.0.0-20241121120954-ac9cc734c153/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= diff --git a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go index 3c0a7391b..ede8f0473 100644 --- a/src/k8s/pkg/k8sd/api/cluster_bootstrap.go +++ b/src/k8s/pkg/k8sd/api/cluster_bootstrap.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path/filepath" "time" apiv1 "github.com/canonical/k8s-snap-api/api/v1" @@ -36,6 +37,12 @@ func (e *Endpoints) postClusterBootstrap(_ state.State, r *http.Request) respons return response.BadRequest(fmt.Errorf("cluster is already bootstrapped")) } + // If not set, leave the default base dir location. + if req.Config.ContainerdBaseDir != "" { + // append k8s-containerd to the given base dir, so we don't flood it with our own folders. + e.provider.Snap().SetContainerdBaseDir(filepath.Join(req.Config.ContainerdBaseDir, "k8s-containerd")) + } + // NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set. ctx, cancel := context.WithTimeout(r.Context(), time.Hour) defer cancel() diff --git a/src/k8s/pkg/k8sd/api/cluster_join.go b/src/k8s/pkg/k8sd/api/cluster_join.go index 3d19997db..9f43625b6 100644 --- a/src/k8s/pkg/k8sd/api/cluster_join.go +++ b/src/k8s/pkg/k8sd/api/cluster_join.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path/filepath" "time" apiv1 "github.com/canonical/k8s-snap-api/api/v1" @@ -11,6 +12,7 @@ import ( "github.com/canonical/k8s/pkg/utils" "github.com/canonical/lxd/lxd/response" "github.com/canonical/microcluster/v3/state" + "gopkg.in/yaml.v2" ) func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Response { @@ -28,6 +30,20 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res return NodeInUse(fmt.Errorf("node %q is part of the cluster", hostname)) } + joinConfig := struct { + // We only care about this field from the entire join config. + containerdBaseDir string `yaml:"containerd-base-dir,omitempty"` + }{} + + if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil { + return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err)) + } + + if joinConfig.containerdBaseDir != "" { + // append k8s-containerd to the given base dir, so we don't flood it with our own folders. + e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd")) + } + config := map[string]string{} // NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set. diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go index eb9909124..d6c6c8a10 100644 --- a/src/k8s/pkg/k8sd/setup/containerd.go +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -173,6 +173,7 @@ func saveSnapContainerdPaths(s snap.Snap) error { "containerd-config-dir": s.ContainerdConfigDir(), "containerd-root-dir": s.ContainerdRootDir(), "containerd-cni-bin-dir": s.CNIBinDir(), + snap.ContainerdBaseDir: s.GetContainerdBaseDir(), } for filename, content := range m { diff --git a/src/k8s/pkg/snap/interface.go b/src/k8s/pkg/snap/interface.go index f2ea533df..b26087f5c 100644 --- a/src/k8s/pkg/snap/interface.go +++ b/src/k8s/pkg/snap/interface.go @@ -39,6 +39,8 @@ type Snap interface { EtcdPKIDir() string // /etc/kubernetes/pki/etcd KubeletRootDir() string // /var/lib/kubelet + SetContainerdBaseDir(baseDir string) // sets the containerd base directory. + GetContainerdBaseDir() string // gets the containerd base directory. ContainerdConfigDir() string // classic confinement: /etc/containerd, strict confinement: /var/snap/k8s/common/etc/containerd ContainerdExtraConfigDir() string // classic confinement: /etc/containerd/conf.d, strict confinement: /var/snap/k8s/common/etc/containerd/conf.d ContainerdRegistryConfigDir() string // classic confinement: /etc/containerd/hosts.d, strict confinement: /var/snap/k8s/common/etc/containerd/hosts.d diff --git a/src/k8s/pkg/snap/mock/mock.go b/src/k8s/pkg/snap/mock/mock.go index 21846253d..544a65471 100644 --- a/src/k8s/pkg/snap/mock/mock.go +++ b/src/k8s/pkg/snap/mock/mock.go @@ -30,6 +30,7 @@ type Mock struct { ContainerdConfigDir string ContainerdExtraConfigDir string ContainerdRegistryConfigDir string + ContainerdBaseDir string ContainerdRootDir string ContainerdSocketDir string ContainerdSocketPath string @@ -131,6 +132,14 @@ func (s *Snap) Hostname() string { return s.Mock.Hostname } +func (s *Snap) SetContainerdBaseDir(baseDir string) { + s.Mock.ContainerdBaseDir = baseDir +} + +func (s *Snap) GetContainerdBaseDir() string { + return s.Mock.ContainerdBaseDir +} + func (s *Snap) ContainerdConfigDir() string { return s.Mock.ContainerdConfigDir } diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index 057d30cde..f5d595379 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -23,6 +23,10 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" ) +const ( + ContainerdBaseDir = ".containerd-base-dir" +) + type SnapOpts struct { SnapInstanceName string SnapDir string @@ -172,6 +176,14 @@ func (s *snap) Hostname() string { return hostname } +func (s *snap) SetContainerdBaseDir(baseDir string) { + s.containerdBaseDir = baseDir +} + +func (s *snap) GetContainerdBaseDir() string { + return s.containerdBaseDir +} + func (s *snap) ContainerdConfigDir() string { return filepath.Join(s.containerdBaseDir, "etc", "containerd") } @@ -346,7 +358,8 @@ func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) er if _, err := os.Stat(s.ContainerdSocketDir()); err == nil { return fmt.Errorf("The path '%s' required for the containerd socket already exists. "+ "This may mean that another service is already using that path, and it conflicts with the k8s snap. "+ - "Please make sure that there is no other service installed that uses the same path, and remove the existing directory.", s.ContainerdSocketDir()) + "Please make sure that there is no other service installed that uses the same path, and remove the existing directory."+ + "(dev-only): You can change the default k8s containerd base path with the containerd-base-dir option in the bootstrap / join-cluster config file.", s.ContainerdSocketDir()) } else if !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Encountered an error while checking '%s': %w", s.ContainerdSocketDir(), err) } diff --git a/tests/integration/templates/bootstrap-containerd-path.yaml b/tests/integration/templates/bootstrap-containerd-path.yaml new file mode 100644 index 000000000..5abfa7878 --- /dev/null +++ b/tests/integration/templates/bootstrap-containerd-path.yaml @@ -0,0 +1,2 @@ +# Contains the bootstrap configuration for the containerd-related paths test. +containerd-base-dir: /home/ubuntu diff --git a/tests/integration/tests/test_cleanup.py b/tests/integration/tests/test_cleanup.py index 784ac6b4a..1540ef5c8 100644 --- a/tests/integration/tests/test_cleanup.py +++ b/tests/integration/tests/test_cleanup.py @@ -2,19 +2,21 @@ # Copyright 2024 Canonical, Ltd. # import logging +import os from typing import List import pytest -from test_util import harness, util +import yaml +from test_util import config, harness, util LOG = logging.getLogger(__name__) CONTAINERD_PATHS = [ "/etc/containerd", - "/opt/cni/bin", "/run/containerd", "/var/lib/containerd", ] +CNI_PATH = "/opt/cni/bin" @pytest.mark.node_count(1) @@ -26,8 +28,61 @@ def test_node_cleanup(instances: List[harness.Instance]): util.remove_k8s_snap(instance) # Check that the containerd-related folders are removed on snap removal. + all_paths = CONTAINERD_PATHS + [CNI_PATH] process = instance.exec( - ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False + ["ls", CNI_PATH, *all_paths], capture_output=True, text=True, check=False ) - for path in CONTAINERD_PATHS: + for path in all_paths: assert f"cannot access '{path}': No such file or directory" in process.stderr + + +@pytest.mark.node_count(2) +@pytest.mark.disable_k8s_bootstrapping() +def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]): + main = instances[0] + joiner = instances[1] + + containerd_path_bootstrap_config = ( + config.MANIFESTS_DIR / "bootstrap-containerd-path.yaml" + ).read_text() + + main.exec( + ["k8s", "bootstrap", "--file", "-"], + input=str.encode(containerd_path_bootstrap_config), + ) + + join_token = util.get_join_token(main, joiner) + joiner.exec( + ["k8s", "join-cluster", join_token, "--file", "-"], + input=str.encode(containerd_path_bootstrap_config), + ) + + boostrap_config = yaml.safe_load(containerd_path_bootstrap_config) + new_containerd_paths = [ + os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p) + for p in CONTAINERD_PATHS + ] + for instance in instances: + # Check that the containerd-related folders are not in the default locations. + process = instance.exec( + ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False + ) + for path in CONTAINERD_PATHS: + assert ( + f"cannot access '{path}': No such file or directory" in process.stderr + ) + + # Check that the containerd-related folders are in the new locations. + # If one of them is missing, this should have a non-zero exit code. + instance.exec(["ls", *new_containerd_paths], check=True) + + for instance in instances: + # Check that the containerd-related folders are not in the new locations after snap removal. + util.remove_k8s_snap(instance) + process = instance.exec( + ["ls", *new_containerd_paths], capture_output=True, text=True, check=False + ) + for path in new_containerd_paths: + assert ( + f"cannot access '{path}': No such file or directory" in process.stderr + )