fix: force Flannel CNI to use KubePrism Kubernetes API endpoint

Fixes #8501 Signed-off-by: Andrey Smirnov <[email protected]>
siderolabs · Apr 2, 2024 · 3195e5d · 3195e5d
1 parent 917043f
commit 3195e5d
Show file tree

Hide file tree

Showing 8 changed files with 472 additions and 332 deletions.
diff --git a/api/resource/definitions/k8s/k8s.proto b/api/resource/definitions/k8s/k8s.proto
@@ -58,6 +58,8 @@ message BootstrapManifestsConfigSpec {
   bool pod_security_policy_enabled = 14;
   bool talos_api_service_enabled = 15;
   repeated string flannel_extra_args = 16;
+  string flannel_kube_service_host = 17;
+  string flannel_kube_service_port = 18;
 }
 
 // ConfigStatusSpec describes status of rendered secrets.

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane.go b/internal/app/machined/pkg/controllers/k8s/control_plane.go
@@ -7,6 +7,7 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/cosi-project/runtime/pkg/controller"
@@ -241,9 +242,15 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					return err
 				}
 
-				var server string
+				var (
+					server                                         string
+					flannelKubeServiceHost, flannelKubeServicePort string
+				)
+
 				if cfgProvider.Machine().Features().KubePrism().Enabled() {
 					server = fmt.Sprintf("https://127.0.0.1:%d", cfgProvider.Machine().Features().KubePrism().Port())
+					flannelKubeServiceHost = "127.0.0.1"
+					flannelKubeServicePort = strconv.Itoa(cfgProvider.Machine().Features().KubePrism().Port())
 				} else {
 					server = cfgProvider.Cluster().Endpoint().String()
 				}
@@ -264,10 +271,12 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					DNSServiceIP:   dnsServiceIP,
 					DNSServiceIPv6: dnsServiceIPv6,
 
-					FlannelEnabled:   cfgProvider.Cluster().Network().CNI().Name() == constants.FlannelCNI,
-					FlannelImage:     images.Flannel,
-					FlannelCNIImage:  images.FlannelCNI,
-					FlannelExtraArgs: cfgProvider.Cluster().Network().CNI().Flannel().ExtraArgs(),
+					FlannelEnabled:         cfgProvider.Cluster().Network().CNI().Name() == constants.FlannelCNI,
+					FlannelImage:           images.Flannel,
+					FlannelCNIImage:        images.FlannelCNI,
+					FlannelExtraArgs:       cfgProvider.Cluster().Network().CNI().Flannel().ExtraArgs(),
+					FlannelKubeServiceHost: flannelKubeServiceHost,
+					FlannelKubeServicePort: flannelKubeServicePort,
 
 					PodSecurityPolicyEnabled: !cfgProvider.Cluster().APIServer().DisablePodSecurityPolicy(),
 

diff --git a/pkg/flannel/gen.go b/pkg/flannel/gen.go
@@ -91,6 +91,7 @@ func marshal(out io.Writer, obj runtime.Object) {
 	m = regexp.MustCompile(`status:\n(  .+\n)+`).ReplaceAll(m, nil)
 
 	m = regexp.MustCompile(`( +)- EXTRA_ARGS_PLACEHOLDER`).ReplaceAll(m, []byte("$1{{- range $$arg := .FlannelExtraArgs }}\n$1- {{ $$arg | json }}\n$1{{- end }}"))
+	m = regexp.MustCompile(`( +)- name: EXTRA_ENV_PLACEHOLDER`).ReplaceAll(m, []byte("$1{{- if .FlannelKubeServiceHost }}\n$1- name: KUBERNETES_SERVICE_HOST\n$1  value: {{ .FlannelKubeServiceHost | json }}\n$1{{- end }}\n$1{{- if .FlannelKubeServicePort }}\n$1- name: KUBERNETES_SERVICE_PORT\n$1  value: {{ .FlannelKubeServicePort | json }}\n$1{{- end }}"))
 
 	fmt.Fprintf(out, "%s---\n", string(m))
 }
@@ -188,6 +189,9 @@ var Template = []byte(`+"`", url)
 			ds.Spec.Template.Spec.Containers[0].Args = append(ds.Spec.Template.Spec.Containers[0].Args,
 				"EXTRA_ARGS_PLACEHOLDER")
 
+			ds.Spec.Template.Spec.Containers[0].Env = append(ds.Spec.Template.Spec.Containers[0].Env,
+				corev1.EnvVar{Name: "EXTRA_ENV_PLACEHOLDER"})
+
 			ds.Spec.Template.Spec.Volumes = xslices.FilterInPlace(ds.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool {
 				return v.Name != "xtables-lock"
 			})

diff --git a/pkg/flannel/template.go b/pkg/flannel/template.go
diff --git a/pkg/machinery/api/resource/definitions/k8s/k8s.pb.go b/pkg/machinery/api/resource/definitions/k8s/k8s.pb.go
diff --git a/pkg/machinery/api/resource/definitions/k8s/k8s_vtproto.pb.go b/pkg/machinery/api/resource/definitions/k8s/k8s_vtproto.pb.go
diff --git a/pkg/machinery/resources/k8s/manifests_config.go b/pkg/machinery/resources/k8s/manifests_config.go
@@ -42,10 +42,12 @@ type BootstrapManifestsConfigSpec struct {
 	DNSServiceIP   string `yaml:"dnsServiceIP" protobuf:"9"`
 	DNSServiceIPv6 string `yaml:"dnsServiceIPv6" protobuf:"10"`
 
-	FlannelEnabled   bool     `yaml:"flannelEnabled" protobuf:"11"`
-	FlannelImage     string   `yaml:"flannelImage" protobuf:"12"`
-	FlannelCNIImage  string   `yaml:"flannelCNIImage" protobuf:"13"`
-	FlannelExtraArgs []string `yaml:"flannelExtraArgs" protobuf:"16"`
+	FlannelEnabled         bool     `yaml:"flannelEnabled" protobuf:"11"`
+	FlannelImage           string   `yaml:"flannelImage" protobuf:"12"`
+	FlannelCNIImage        string   `yaml:"flannelCNIImage" protobuf:"13"`
+	FlannelExtraArgs       []string `yaml:"flannelExtraArgs" protobuf:"16"`
+	FlannelKubeServiceHost string   `yaml:"flannelKubeServiceHost" protobuf:"17"`
+	FlannelKubeServicePort string   `yaml:"flannelKubeServicePort" protobuf:"18"`
 
 	PodSecurityPolicyEnabled bool `yaml:"podSecurityPolicyEnabled" protobuf:"14"`
 

diff --git a/website/content/v1.7/reference/api.md b/website/content/v1.7/reference/api.md
@@ -2044,6 +2044,8 @@ BootstrapManifestsConfigSpec is configuration for bootstrap manifests.
 | pod_security_policy_enabled | [bool](#bool) |  |  |
 | talos_api_service_enabled | [bool](#bool) |  |  |
 | flannel_extra_args | [string](#string) | repeated |  |
+| flannel_kube_service_host | [string](#string) |  |  |
+| flannel_kube_service_port | [string](#string) |  |  |