kube: resolve Kubernetes cluster domain (#43584) (#43633)

This PR enhances teleport's intelligence by no longer assuming the cluster domain is `cluster.local`. Since many clusters use a non-default cluster domain, this assumption can disrupt app discovery access. Fixes #39007 Signed-off-by: Tiago Silva <[email protected]>
gravitational · Jul 8, 2024 · f3db7d0 · f3db7d0
1 parent 08c3939
commit f3db7d0
Show file tree

Hide file tree

Showing 7 changed files with 147 additions and 22 deletions.
diff --git a/docs/pages/reference/helm-reference/teleport-kube-agent.mdx b/docs/pages/reference/helm-reference/teleport-kube-agent.mdx
@@ -524,6 +524,17 @@ You can specify multiple selectors by including additional list elements.
   Once `appResources` is set, you can dynamically register application with `tsh` by following [this guide](../../application-access/guides/dynamic-registration.mdx).
 </Admonition>
 
+## `clusterDomain`
+
+| Type | Default |
+|------|---------|
+| `string` | `"cluster.local"` |
+
+`clusterDomain` sets the domain name used by the Kubernetes cluster. This value is used to build the
+FQDN application URIs. For example, if the cluster domain is `anything.local`, the agent will proxy the application
+`my-app` running in the `default` namespace at `http://my-app.default.svc.anything.local`. You must manually set this value
+to match your cluster domain if it is different from the default value `cluster.local`.
+
 ## `awsDatabases`
 
 <Admonition type="tip" title="AWS database auto-discovery">

diff --git a/examples/chart/teleport-kube-agent/templates/deployment.yaml b/examples/chart/teleport-kube-agent/templates/deployment.yaml
@@ -136,6 +136,10 @@ spec:
         - name: TELEPORT_EXT_UPGRADER_VERSION
           value: {{ include "teleport-kube-agent.version" . }}
         {{- end }}
+        {{- if .Values.clusterDomain }}
+        - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+          value: {{ .Values.clusterDomain | quote }}
+        {{- end }}
         {{- if (gt (len .Values.extraEnv) 0) }}
           {{- toYaml .Values.extraEnv | nindent 8 }}
         {{- end }}

diff --git a/examples/chart/teleport-kube-agent/tests/__snapshot__/deployment_test.yaml.snap b/examples/chart/teleport-kube-agent/tests/__snapshot__/deployment_test.yaml.snap
@@ -30,6 +30,8 @@ sets Deployment annotations when specified if action is Upgrade:
             env:
             - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
               value: "true"
+            - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+              value: cluster.local
             image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
             imagePullPolicy: IfNotPresent
             livenessProbe:
@@ -103,6 +105,8 @@ sets Deployment labels when specified if action is Upgrade:
           env:
           - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
             value: "true"
+          - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+            value: cluster.local
           image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
           imagePullPolicy: IfNotPresent
           livenessProbe:
@@ -163,6 +167,8 @@ sets Pod annotations when specified if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -223,6 +229,8 @@ sets Pod labels when specified if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -300,6 +308,8 @@ should add emptyDir for data when existingDataVolume is not set if action is Upg
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -361,6 +371,8 @@ should add insecureSkipProxyTLSVerify to args when set in values if action is Up
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -421,6 +433,8 @@ should correctly configure existingDataVolume when set if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -479,6 +493,8 @@ should expose diag port if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -551,6 +567,8 @@ should have multiple replicas when replicaCount is set (using .replicaCount, dep
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -623,6 +641,8 @@ should have multiple replicas when replicaCount is set (using highAvailability.r
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -683,6 +703,8 @@ should have one replica when replicaCount is not set if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -743,6 +765,8 @@ should mount extraVolumes and extraVolumeMounts if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -808,6 +832,8 @@ should mount tls.existingCASecretName and set environment when set in values if
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       - name: SSL_CERT_FILE
         value: /etc/teleport-tls-ca/ca.pem
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
@@ -876,6 +902,8 @@ should mount tls.existingCASecretName and set extra environment when set in valu
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       - name: HTTPS_PROXY
         value: http://username:[email protected]:3128
       - name: SSL_CERT_FILE
@@ -946,6 +974,8 @@ should provision initContainer correctly when set in values if action is Upgrade
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1042,6 +1072,8 @@ should set SecurityContext if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1122,6 +1154,8 @@ should set affinity when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1182,6 +1216,8 @@ should set default serviceAccountName when not set in values if action is Upgrad
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1253,6 +1289,8 @@ should set environment when extraEnv set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       - name: HTTPS_PROXY
         value: http://username:[email protected]:3128
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
@@ -1315,6 +1353,8 @@ should set image and tag correctly if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:12.2.1
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1375,6 +1415,8 @@ should set imagePullPolicy when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: Always
       livenessProbe:
@@ -1435,6 +1477,8 @@ should set nodeSelector if set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1497,6 +1541,8 @@ should set not set priorityClassName when not set in values if action is Upgrade
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1569,6 +1615,8 @@ should set preferred affinity when more than one replica is used if action is Up
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1629,6 +1677,8 @@ should set priorityClassName when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1690,6 +1740,8 @@ should set probeTimeoutSeconds when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1760,6 +1812,8 @@ should set required affinity when highAvailability.requireAntiAffinity is set if
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1820,6 +1874,8 @@ should set resources when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1887,6 +1943,8 @@ should set serviceAccountName when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1947,6 +2005,8 @@ should set tolerations when set in values if action is Upgrade:
       env:
       - name: TELEPORT_INSTALL_METHOD_HELM_KUBE_AGENT
         value: "true"
+      - name: TELEPORT_KUBE_CLUSTER_DOMAIN
+        value: cluster.local
       image: public.ecr.aws/gravitational/teleport-distroless:14.3.21
       imagePullPolicy: IfNotPresent
       livenessProbe:

diff --git a/examples/chart/teleport-kube-agent/values.yaml b/examples/chart/teleport-kube-agent/values.yaml
@@ -52,6 +52,12 @@ apps: []
 #      "*": "*"
 appResources: []
 
+# clusterDomain(string) -- sets the domain name used by the Kubernetes cluster. This value is used to build the
+# FQDN application URIs. For example, if the cluster domain is `anything.local`, the agent will proxy the application
+# `myapp` running in the `default` namespace at `http://myapp.default.svc.anything.local`. You must manually set this value
+# to match your cluster domain if it is different from the default value `cluster.local`.
+clusterDomain: "cluster.local"
+
 ################################################################
 # Values that must be provided if Database access is enabled.
 ################################################################

diff --git a/lib/services/app.go b/lib/services/app.go
@@ -20,7 +20,9 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
+	"sync"
 
 	"github.com/gravitational/trace"
 	corev1 "k8s.io/api/core/v1"
@@ -177,7 +179,7 @@ func UnmarshalAppServer(data []byte, opts ...MarshalOption) (types.AppServer, er
 // It transforms service fields and annotations into appropriate Teleport app fields.
 // Service labels are copied to app labels.
 func NewApplicationFromKubeService(service corev1.Service, clusterName, protocol string, port corev1.ServicePort) (types.Application, error) {
-	appURI := buildAppURI(protocol, getServiceFQDN(service), port.Port)
+	appURI := buildAppURI(protocol, GetServiceFQDN(service), port.Port)
 
 	rewriteConfig, err := getAppRewriteConfig(service.GetAnnotations())
 	if err != nil {
@@ -211,14 +213,15 @@ func NewApplicationFromKubeService(service corev1.Service, clusterName, protocol
 	return app, nil
 }
 
-func getServiceFQDN(s corev1.Service) string {
+// GetServiceFQDN returns the fully qualified domain name for the service.
+func GetServiceFQDN(service corev1.Service) string {
 	// If service type is ExternalName it points to external DNS name, to keep correct
 	// HOST for HTTP requests we return already final external DNS name.
 	// https://kubernetes.io/docs/concepts/services-networking/service/#externalname
-	if s.Spec.Type == corev1.ServiceTypeExternalName {
-		return s.Spec.ExternalName
+	if service.Spec.Type == corev1.ServiceTypeExternalName {
+		return service.Spec.ExternalName
 	}
-	return fmt.Sprintf("%s.%s.svc.cluster.local", s.GetName(), s.GetNamespace())
+	return fmt.Sprintf("%s.%s.svc.%s", service.GetName(), service.GetNamespace(), clusterDomainResolver())
 }
 
 func buildAppURI(protocol, serviceFQDN string, port int32) string {
@@ -289,3 +292,23 @@ func getAppLabels(serviceLabels map[string]string, clusterName string) (map[stri
 
 	return result, nil
 }
+
+var (
+	// clusterDomainResolver is a function that resolves the cluster domain once and caches the result.
+	// It's used to lazily resolve the cluster domain from the env var "TELEPORT_KUBE_CLUSTER_DOMAIN" or fallback to
+	// a default value.
+	// It's only used when agent is running in the Kubernetes cluster.
+	clusterDomainResolver = sync.OnceValue[string](getClusterDomain)
+)
+
+const (
+	// teleportKubeClusterDomain is the environment variable that specifies the cluster domain.
+	teleportKubeClusterDomain = "TELEPORT_KUBE_CLUSTER_DOMAIN"
+)
+
+func getClusterDomain() string {
+	if envDomain := os.Getenv(teleportKubeClusterDomain); envDomain != "" {
+		return envDomain
+	}
+	return "cluster.local"
+}