From a8bd0f99a74f370e0a9bc4ed380ab9dc4bed2a74 Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Fri, 13 Dec 2024 17:40:27 +0000 Subject: [PATCH] Kubernetes App Auto Discovery: improve protocol detection Kubernetes App Auto Discovery iterates over all Services and tries to auto enroll them as Teleport Applications. During this process, it tries to guess the Service's port protocol to ensure we add the application only if it's either an HTTP or HTTPS capable Service. When there's not annotation configuration (which are teleport specific), we try to infer from the Service's ports. When that doesn't work out, the teleport-agent issues an HTTP HEAD request against the port. This way we detect whether the service can answer HTTP or HTTPS. This PR changes the way teleport infers the protocol using the Service's Port. It was checking for HTTPS (checking for port number and port name), then it did a HTTP HEAD request and then it was checking for HTTP (checking port number and port name). This PR changes 4 things: - checks the port, the node port and the target port against well known ports (443, 80, 8080) - checks the name of the port in bother Port.Name and Port.TargetPort - tries to do HTTPS and HTTP checks before trying an HTTP request - decreases the HTTP request timeout from 5s to 500ms With a demo cluster with 2700+ Services, the reconciliation time decreased from 2m to something very close to 0s. --- lib/srv/discovery/fetchers/kube_services.go | 49 ++++++++++++--------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/lib/srv/discovery/fetchers/kube_services.go b/lib/srv/discovery/fetchers/kube_services.go index b36fa57839c53..3574e0a31a851 100644 --- a/lib/srv/discovery/fetchers/kube_services.go +++ b/lib/srv/discovery/fetchers/kube_services.go @@ -102,7 +102,7 @@ func isInternalKubeService(s v1.Service) bool { s.GetNamespace() == metav1.NamespacePublic } -func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error) { +func (f *KubeAppFetcher) getServices(ctx context.Context, discoveryType string) ([]v1.Service, error) { var result []v1.Service nextToken := "" namespaceFilter := func(ns string) bool { @@ -125,6 +125,17 @@ func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error) // Namespace is not in the list of namespaces to fetch or it's an internal service continue } + + // Skip service if it has type annotation and it's not the expected type. + if v, ok := s.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != discoveryType { + continue + } + + // If the service is marked with the ignore annotation, skip it. + if v := s.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" { + continue + } + match, _, err := services.MatchLabels(f.FilterLabels, s.Labels) if err != nil { return nil, trace.Wrap(err) @@ -150,7 +161,7 @@ const ( // Get fetches Kubernetes apps from the cluster func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, error) { - kubeServices, err := f.getServices(ctx) + kubeServices, err := f.getServices(ctx, types.KubernetesMatchersApp) if err != nil { return nil, trace.Wrap(err) } @@ -159,7 +170,7 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er // Both services and ports inside services are processed in parallel to minimize time. // We also set limit to prevent potential spike load on a cluster in case there are a lot of services. g, _ := errgroup.WithContext(ctx) - g.SetLimit(10) + g.SetLimit(20) // Convert services to resources var ( @@ -168,17 +179,6 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er ) for _, service := range kubeServices { service := service - - // Skip service if it has type annotation and it's not 'app' - if v, ok := service.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != types.KubernetesMatchersApp { - continue - } - - // If the service is marked with the ignore annotation, skip it. - if v := service.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" { - continue - } - g.Go(func() error { protocolAnnotation := service.GetAnnotations()[types.DiscoveryProtocolLabel] @@ -256,9 +256,9 @@ func (f *KubeAppFetcher) String() string { // by protocol checker. It is used when no explicit annotation for port's protocol was provided. // - If port's AppProtocol specifies `http` or `https` we return it // - If port's name is `https` or number is 443 we return `https` +// - If port's name is `http` or number is 80 or 8080, we return `http` // - If protocol checker is available it will perform HTTP request to the service fqdn trying to find out protocol. If it // gives us result `http` or `https` we return it -// - If port's name is `http` or number is 80 or 8080, we return `http` func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolChecker) string { if port.AppProtocol != nil { switch p := strings.ToLower(*port.AppProtocol); p { @@ -267,10 +267,19 @@ func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolC } } - if port.Port == 443 || strings.EqualFold(port.Name, protoHTTPS) { + if strings.EqualFold(port.Name, protoHTTPS) || strings.EqualFold(port.TargetPort.StrVal, protoHTTPS) || + port.Port == 443 || port.NodePort == 443 || port.TargetPort.IntVal == 443 { + return protoHTTPS } + if strings.EqualFold(port.Name, protoHTTP) || strings.EqualFold(port.TargetPort.StrVal, protoHTTP) || + port.Port == 80 || port.NodePort == 80 || port.TargetPort.IntVal == 80 || + port.Port == 8080 || port.NodePort == 8080 || port.TargetPort.IntVal == 8080 { + + return protoHTTP + } + if pc != nil { result := pc.CheckProtocol(fmt.Sprintf("%s:%d", serviceFQDN, port.Port)) if result != protoTCP { @@ -278,10 +287,6 @@ func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolC } } - if port.Port == 80 || port.Port == 8080 || strings.EqualFold(port.Name, protoHTTP) { - return protoHTTP - } - return protoTCP } @@ -327,7 +332,9 @@ func NewProtoChecker(insecureSkipVerify bool) *ProtoChecker { p := &ProtoChecker{ InsecureSkipVerify: insecureSkipVerify, client: &http.Client{ - Timeout: 5 * time.Second, + // This is a best-effort scenario, where teleport tries to guess which protocol is being used. + // Ideally it should either be inferred by the Service's ports or explicitly configured by using annotations on the service. + Timeout: 500 * time.Millisecond, Transport: &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: insecureSkipVerify,