From 1f381fe85432c0cbcd03d52a7943f2e618737de0 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 26 Jul 2024 12:17:38 -0400 Subject: [PATCH] add new disableDefaultCNI field --- apis/v1beta1/microk8sconfig_types.go | 4 + ...trap.cluster.x-k8s.io_microk8sconfigs.yaml | 3 + ...ster.x-k8s.io_microk8sconfigtemplates.yaml | 3 + .../cloudinit/cloudinit_common_test.go | 76 +++++++++++++++++++ controllers/cloudinit/controlplane_init.go | 10 ++- .../cloudinit/controlplane_init_test.go | 2 + controllers/cloudinit/controlplane_join.go | 10 ++- controllers/cloudinit/embed.go | 3 + .../scripts/00-disable-default-cni.sh | 4 + controllers/microk8sconfig_controller.go | 2 + 10 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 controllers/cloudinit/scripts/00-disable-default-cni.sh diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go index 46709bc..782f9ec 100644 --- a/apis/v1beta1/microk8sconfig_types.go +++ b/apis/v1beta1/microk8sconfig_types.go @@ -81,6 +81,10 @@ type InitConfiguration struct { // +kubebuilder:default:=stable RiskLevel string `json:"riskLevel,omitempty"` + // Whether or not to use the default CNI + // +optional + DisableDefaultCNI bool `json:"useDefaultCNI,omitempty"` + // The snap store proxy domain // +optional SnapstoreProxyDomain string `json:"snapstoreProxyDomain,omitempty"` diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml index 9678ad2..ec76a15 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml @@ -177,6 +177,9 @@ spec: snapstoreProxyId: description: The snap store proxy ID type: string + useDefaultCNI: + description: Whether or not to use the default CNI + type: boolean type: object type: object status: diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml index fae34e3..149cd4d 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml @@ -188,6 +188,9 @@ spec: snapstoreProxyId: description: The snap store proxy ID type: string + useDefaultCNI: + description: Whether or not to use the default CNI + type: boolean type: object type: object type: object diff --git a/controllers/cloudinit/cloudinit_common_test.go b/controllers/cloudinit/cloudinit_common_test.go index d2cf406..c23fae4 100644 --- a/controllers/cloudinit/cloudinit_common_test.go +++ b/controllers/cloudinit/cloudinit_common_test.go @@ -382,4 +382,80 @@ func TestCloudConfigInput(t *testing.T) { }) } }) + + t.Run("DisableDefaultCNI", func(t *testing.T) { + for _, tc := range []struct { + name string + makeCloudConfig func() (*cloudinit.CloudConfig, error) + }{ + { + name: "ControlPlaneInit", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + DisableDefaultCNI: true, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + { + name: "ControlPlaneJoin", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + DisableDefaultCNI: true, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c, err := tc.makeCloudConfig() + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(c.RunCommands).To(ContainElement(`/capi-scripts/00-disable-default-cni.sh`)) + }) + } + }) + + t.Run("DefaultCNI", func(t *testing.T) { + for _, tc := range []struct { + name string + makeCloudConfig func() (*cloudinit.CloudConfig, error) + }{ + { + name: "ControlPlaneInit", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + DisableDefaultCNI: false, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + { + name: "ControlPlaneJoin", + makeCloudConfig: func() (*cloudinit.CloudConfig, error) { + return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + DisableDefaultCNI: false, + KubernetesVersion: "v1.25.0", + Token: strings.Repeat("a", 32), + TokenTTL: 100, + }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c, err := tc.makeCloudConfig() + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(c.RunCommands).NotTo(ContainElement(`/capi-scripts/00-disable-default-cni.sh`)) + }) + } + }) } diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index 4eb1123..1de09e4 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -57,6 +57,8 @@ type ControlPlaneInitInput struct { Confinement string // RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels. RiskLevel string + // DisableDefaultCNI specifies whether to disable the default CNI plugin. + DisableDefaultCNI bool // SnapstoreProxyDomain specifies the domain of the snapstore proxy if one is to be used. SnapstoreProxyDomain string // SnapstoreProxyId specifies the snapstore proxy ID if one is to be used. @@ -141,7 +143,13 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy), fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId), scriptPath(disableHostServicesScript), - fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), + fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs)) + + if input.DisableDefaultCNI { + cloudConfig.RunCommands = append(cloudConfig.RunCommands, scriptPath(disableDefaultCNIScript)) + } + + cloudConfig.RunCommands = append(cloudConfig.RunCommands, fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), scriptPath(configureKubeletScript), scriptPath(waitAPIServerScript), diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index b4cbaa1..c7c658e 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -38,6 +38,7 @@ func TestControlPlaneInit(t *testing.T) { IPinIP: true, Token: strings.Repeat("a", 32), TokenTTL: 10000, + DisableDefaultCNI: true, Confinement: "classic", }) g.Expect(err).NotTo(HaveOccurred()) @@ -48,6 +49,7 @@ func TestControlPlaneInit(t *testing.T) { `/capi-scripts/00-configure-snapstore-proxy.sh "" ""`, `/capi-scripts/00-disable-host-services.sh`, `/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`, + `/capi-scripts/00-disable-default-cni.sh`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, `/capi-scripts/10-configure-kubelet.sh`, `/capi-scripts/50-wait-apiserver.sh`, diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 7c936dd..8ef3dc0 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -53,6 +53,8 @@ type ControlPlaneJoinInput struct { Confinement string // RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels. RiskLevel string + // DisableDefaultCNI specifies whether to use the default CNI plugin. + DisableDefaultCNI bool // SnapstoreProxyDomain specifies the domain of the snapstore proxy if one is to be used. SnapstoreProxyDomain string // SnapstoreProxyId specifies the snapstore proxy ID if one is to be used. @@ -123,7 +125,13 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy), fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId), scriptPath(disableHostServicesScript), - fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), + fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs)) + + if input.DisableDefaultCNI { + cloudConfig.RunCommands = append(cloudConfig.RunCommands, scriptPath(disableDefaultCNIScript)) + } + + cloudConfig.RunCommands = append(cloudConfig.RunCommands, fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), scriptPath(configureKubeletScript), scriptPath(waitAPIServerScript), diff --git a/controllers/cloudinit/embed.go b/controllers/cloudinit/embed.go index 39fa77e..00ffc65 100644 --- a/controllers/cloudinit/embed.go +++ b/controllers/cloudinit/embed.go @@ -43,6 +43,9 @@ const ( // disableHostServicesScript disables services like containerd or kubelet from the host OS image. disableHostServicesScript script = "00-disable-host-services.sh" + // disableDefaultCNIScript disables the default CNI plugin. + disableDefaultCNIScript script = "00-disable-default-cni.sh" + // installMicroK8sScript installs MicroK8s on the host. installMicroK8sScript script = "00-install-microk8s.sh" diff --git a/controllers/cloudinit/scripts/00-disable-default-cni.sh b/controllers/cloudinit/scripts/00-disable-default-cni.sh new file mode 100644 index 0000000..9d13aff --- /dev/null +++ b/controllers/cloudinit/scripts/00-disable-default-cni.sh @@ -0,0 +1,4 @@ +#!/bin/bash -xe + +microk8s kubectl delete -f /var/snap/microk8s/current/args/cni-network/cni.yaml +mv /var/snap/microk8s/current/args/cni-network/cni.yaml /var/snap/microk8s/current/args/cni-network/calico.yaml.old \ No newline at end of file diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index 6aa10e6..b836e7a 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -314,6 +314,7 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte SnapstoreProxyId: microk8sConfig.Spec.InitConfiguration.SnapstoreProxyId, Confinement: microk8sConfig.Spec.InitConfiguration.Confinement, RiskLevel: microk8sConfig.Spec.InitConfiguration.RiskLevel, + DisableDefaultCNI: microk8sConfig.Spec.InitConfiguration.DisableDefaultCNI, ExtraWriteFiles: cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles), ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs, SnapstoreHTTPProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy, @@ -416,6 +417,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con SnapstoreProxyDomain: microk8sConfig.Spec.InitConfiguration.SnapstoreProxyDomain, SnapstoreProxyId: microk8sConfig.Spec.InitConfiguration.SnapstoreProxyId, RiskLevel: microk8sConfig.Spec.InitConfiguration.RiskLevel, + DisableDefaultCNI: microk8sConfig.Spec.InitConfiguration.DisableDefaultCNI, Confinement: microk8sConfig.Spec.InitConfiguration.Confinement, ExtraWriteFiles: cloudinit.WriteFilesFromAPI(microk8sConfig.Spec.InitConfiguration.ExtraWriteFiles), ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs,