diff --git a/internal/annotations/annotations.go b/internal/annotations/annotations.go index 44467a6e02..55b1c89d2c 100644 --- a/internal/annotations/annotations.go +++ b/internal/annotations/annotations.go @@ -41,35 +41,36 @@ const ( AnnotationPrefix = "konghq.com" - ConfigurationKey = "/override" - PluginsKey = "/plugins" - ProtocolKey = "/protocol" - ProtocolsKey = "/protocols" - ClientCertKey = "/client-cert" - StripPathKey = "/strip-path" - PathKey = "/path" - HTTPSRedirectCodeKey = "/https-redirect-status-code" - PreserveHostKey = "/preserve-host" - RegexPriorityKey = "/regex-priority" - HostHeaderKey = "/host-header" - MethodsKey = "/methods" - SNIsKey = "/snis" - RequestBuffering = "/request-buffering" - ResponseBuffering = "/response-buffering" - HostAliasesKey = "/host-aliases" - RegexPrefixKey = "/regex-prefix" - ConnectTimeoutKey = "/connect-timeout" - WriteTimeoutKey = "/write-timeout" - ReadTimeoutKey = "/read-timeout" - RetriesKey = "/retries" - HeadersKey = "/headers" - HeadersSeparatorKey = "/headers-separator" - PathHandlingKey = "/path-handling" - UserTagKey = "/tags" - RewriteURIKey = "/rewrite" - TLSVerifyKey = "/tls-verify" - TLSVerifyDepthKey = "/tls-verify-depth" - CACertificatesKey = "/ca-certificates" + ConfigurationKey = "/override" + PluginsKey = "/plugins" + ProtocolKey = "/protocol" + ProtocolsKey = "/protocols" + ClientCertKey = "/client-cert" + StripPathKey = "/strip-path" + PathKey = "/path" + HTTPSRedirectCodeKey = "/https-redirect-status-code" + PreserveHostKey = "/preserve-host" + RegexPriorityKey = "/regex-priority" + HostHeaderKey = "/host-header" + MethodsKey = "/methods" + SNIsKey = "/snis" + RequestBuffering = "/request-buffering" + ResponseBuffering = "/response-buffering" + HostAliasesKey = "/host-aliases" + RegexPrefixKey = "/regex-prefix" + ConnectTimeoutKey = "/connect-timeout" + WriteTimeoutKey = "/write-timeout" + ReadTimeoutKey = "/read-timeout" + RetriesKey = "/retries" + HeadersKey = "/headers" + HeadersSeparatorKey = "/headers-separator" + PathHandlingKey = "/path-handling" + UserTagKey = "/tags" + RewriteURIKey = "/rewrite" + TLSVerifyKey = "/tls-verify" + TLSVerifyDepthKey = "/tls-verify-depth" + CACertificatesFromSecretKey = "/ca-certificates-from-secret" + CACertificatesFromConfigMapKey = "/ca-certificates-from-configmap" // GatewayClassUnmanagedKey is an annotation used on a Gateway resource to // indicate that the GatewayClass should be reconciled according to unmanaged @@ -395,10 +396,18 @@ func ExtractTLSVerifyDepth(anns map[string]string) (int, bool) { return depth, true } -// ExtractCACertificates extracts the ca-certificates secret names from the annotation. +// ExtractCACertificatesFromSecrets extracts the ca-certificates secret names from the annotation. // It expects a comma-separated list of certificate names. -func ExtractCACertificates(anns map[string]string) []string { - s, ok := anns[AnnotationPrefix+CACertificatesKey] +func ExtractCACertificatesFromSecrets(anns map[string]string) []string { + s, ok := anns[AnnotationPrefix+CACertificatesFromSecretKey] + if !ok { + return nil + } + return extractCommaDelimitedStrings(s) +} + +func ExtractCACertificatesFromConfigMap(anns map[string]string) []string { + s, ok := anns[AnnotationPrefix+CACertificatesFromConfigMapKey] if !ok { return nil } @@ -448,11 +457,11 @@ func SetTLSVerify(anns map[string]string, value bool) { // SetCACertificates merge the ca-certificates secret names into the already existing CA certificates set via annotation. func SetCACertificates(anns map[string]string, certificates []string) { - existingCACerts := anns[AnnotationPrefix+CACertificatesKey] + existingCACerts := anns[AnnotationPrefix+CACertificatesFromConfigMapKey] if existingCACerts == "" { - anns[AnnotationPrefix+CACertificatesKey] = strings.Join(certificates, ",") + anns[AnnotationPrefix+CACertificatesFromConfigMapKey] = strings.Join(certificates, ",") } else { - anns[AnnotationPrefix+CACertificatesKey] = existingCACerts + "," + strings.Join(certificates, ",") + anns[AnnotationPrefix+CACertificatesFromConfigMapKey] = existingCACerts + "," + strings.Join(certificates, ",") } } @@ -461,6 +470,7 @@ func SetHostHeader(anns map[string]string, value string) { anns[AnnotationPrefix+HostHeaderKey] = value } +// SetProtocol sets the protocol annotation value. func SetProtocol(anns map[string]string, value string) { anns[AnnotationPrefix+ProtocolKey] = value } diff --git a/internal/annotations/annotations_test.go b/internal/annotations/annotations_test.go index 61b2844783..b92a16054c 100644 --- a/internal/annotations/annotations_test.go +++ b/internal/annotations/annotations_test.go @@ -1106,19 +1106,19 @@ func TestExtractTLSVerifyDepth(t *testing.T) { } func TestExtractCACertificates(t *testing.T) { - v := ExtractCACertificates(nil) + v := ExtractCACertificatesFromSecrets(nil) assert.Empty(t, v) - v = ExtractCACertificates(map[string]string{}) + v = ExtractCACertificatesFromSecrets(map[string]string{}) assert.Empty(t, v) - v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: "foo,bar"}) + v = ExtractCACertificatesFromSecrets(map[string]string{AnnotationPrefix + CACertificatesFromSecretKey: "foo,bar"}) assert.Equal(t, []string{"foo", "bar"}, v, "expected to split by comma") - v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: " foo, bar ,baz "}) + v = ExtractCACertificatesFromSecrets(map[string]string{AnnotationPrefix + CACertificatesFromSecretKey: " foo, bar ,baz "}) assert.Equal(t, []string{"foo", "bar", "baz"}, v, "expected to trim spaces") - v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: "foo, bar, "}) + v = ExtractCACertificatesFromSecrets(map[string]string{AnnotationPrefix + CACertificatesFromSecretKey: "foo, bar, "}) assert.Equal(t, []string{"foo", "bar"}, v, "expected to ignore empty values") } diff --git a/internal/dataplane/translator/ingressrules.go b/internal/dataplane/translator/ingressrules.go index 1a237e0a2c..d1ccf54021 100644 --- a/internal/dataplane/translator/ingressrules.go +++ b/internal/dataplane/translator/ingressrules.go @@ -16,6 +16,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/annotations" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/failures" "github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/logging" "github.com/kong/kubernetes-ingress-controller/v3/internal/store" "github.com/kong/kubernetes-ingress-controller/v3/internal/util" @@ -86,17 +87,17 @@ func (ir *ingressRules) populateServices( } for _, k8sService := range k8sServices { - // TODO: comment - ir.handleBackendTLSPolices(logger, s, k8sService, failuresCollector, translatedObjectsCollector) - // at this point we know the Kubernetes service itself is valid and can be // used for traffic, so cache it amongst the kong Services k8s services. service.K8sServices[fmt.Sprintf("%s/%s", k8sService.Namespace, k8sService.Name)] = k8sService - // extract client certificates intended for use by the service + // convert the backendTLSPolicy targeting the service to the proper set of annotations. + ir.handleBackendTLSPolices(logger, s, k8sService, failuresCollector, translatedObjectsCollector) + + // extract client certificates intended for use by the service. ir.handleServiceClientCertificates(s, k8sService, &service, failuresCollector) - // extract CA certificates intended for use by the service + // extract CA certificates intended for use by the service. ir.handleServiceCACertificates(s, k8sService, &service, failuresCollector) } service.Tags = ir.generateKongServiceTags(k8sServices, service, logger) @@ -138,15 +139,13 @@ func (ir *ingressRules) handleBackendTLSPolices( } annotations.SetTLSVerify(k8sService.Annotations, true) - - caCerts := []string{} - for _, cert := range policy.Spec.Validation.CACertificateRefs { - caCerts = append(caCerts, string(cert.Name)) - } - - annotations.SetCACertificates(k8sService.Annotations, caCerts) annotations.SetHostHeader(k8sService.Annotations, string(policy.Spec.Validation.Hostname)) annotations.SetProtocol(k8sService.Annotations, "https") + annotations.SetCACertificates(k8sService.Annotations, + lo.Map(policy.Spec.Validation.CACertificateRefs, func(ref gatewayapi.LocalObjectReference, _ int) string { + return string(ref.Name) + }), + ) } func (ir *ingressRules) handleServiceClientCertificates( @@ -189,8 +188,9 @@ func (ir *ingressRules) handleServiceCACertificates( k *kongstate.Service, collector *failures.ResourceFailuresCollector, ) { - certificates := annotations.ExtractCACertificates(service.Annotations) - if len(certificates) == 0 { + Secretcertificates := annotations.ExtractCACertificatesFromSecrets(service.Annotations) + configMapCertificates := annotations.ExtractCACertificatesFromConfigMap(service.Annotations) + if len(Secretcertificates)+len(configMapCertificates) == 0 { // No CA certificates to process. return } @@ -214,8 +214,8 @@ func (ir *ingressRules) handleServiceCACertificates( return } - // Process each CA certificate and add it to the Kong Service. - for _, certificate := range certificates { + // Process each CA certificate from secret and add it to the Kong Service. + for _, certificate := range Secretcertificates { secretKey := service.Namespace + "/" + certificate secret, err := s.GetSecret(service.Namespace, certificate) if err != nil { @@ -225,10 +225,31 @@ func (ir *ingressRules) handleServiceCACertificates( continue } - certID, ok := secret.Data["id"] + certID, ok := secret.Data["ca.crt"] + if !ok { + collector.PushResourceFailure( + fmt.Sprintf("Invalid CA certificate '%s': missing 'ca.crt' field in data", secretKey), secret, service, + ) + continue + } + k.CACertificates = append(k.CACertificates, lo.ToPtr(string(certID))) + } + + // Process each CA certificate from ConfigMap and add it to the Kong Service. + for _, certificate := range configMapCertificates { + configmapKey := service.Namespace + "/" + certificate + configMap, err := s.GetConfigMap(service.Namespace, certificate) + if err != nil { + collector.PushResourceFailure( + fmt.Sprintf("Failed to fetch configmap for CA Certificate '%s': %v", configmapKey, err), service, + ) + continue + } + + certID, ok := configMap.Data["ca.crt"] if !ok { collector.PushResourceFailure( - fmt.Sprintf("Invalid CA certificate '%s': missing 'id' field in data", secretKey), secret, service, + fmt.Sprintf("Invalid CA certificate '%s': missing 'ca.crt' field in data", configmapKey), configMap, service, ) continue } diff --git a/internal/dataplane/translator/ingressrules_test.go b/internal/dataplane/translator/ingressrules_test.go index 368acc398d..c3583c74d0 100644 --- a/internal/dataplane/translator/ingressrules_test.go +++ b/internal/dataplane/translator/ingressrules_test.go @@ -654,9 +654,9 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-1,ca-2", + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-1,ca-2", }, }, }, @@ -717,8 +717,8 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-1,ca-2", + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-1,ca-2", }, }, }, @@ -773,9 +773,9 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-not-existing", + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-not-existing", }, }, }, @@ -807,9 +807,9 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-1", + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-1", }, }, }, @@ -853,9 +853,9 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-1", - annotations.AnnotationPrefix + annotations.ProtocolKey: "grpc", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-1", + annotations.AnnotationPrefix + annotations.ProtocolKey: "grpc", }, }, }, @@ -900,8 +900,8 @@ func TestPopulateServices(t *testing.T) { Name: "s-1", Namespace: "test-namespace", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca-1", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca-1", }, }, }, diff --git a/internal/dataplane/translator/translator_test.go b/internal/dataplane/translator/translator_test.go index 931971a4b3..b39787533b 100644 --- a/internal/dataplane/translator/translator_test.go +++ b/internal/dataplane/translator/translator_test.go @@ -5295,10 +5295,10 @@ func TestTranslator_IngressUpstreamTLSVerification(t *testing.T) { Name: "svc", Namespace: "ns", Annotations: map[string]string{ - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.TLSVerifyDepthKey: "2", - annotations.AnnotationPrefix + annotations.CACertificatesKey: "ca", - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.TLSVerifyDepthKey: "2", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: "ca", + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", }, }, Spec: corev1.ServiceSpec{ diff --git a/internal/store/store.go b/internal/store/store.go index 0f4caea3fe..bfd99a3a50 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -64,6 +64,7 @@ type Storer interface { // Core Kubernetes resources. GetSecret(namespace, name string) (*corev1.Secret, error) + GetConfigMap(namespace, name string) (*corev1.ConfigMap, error) GetService(namespace, name string) (*corev1.Service, error) GetEndpointSlicesForService(namespace, name string) ([]*discoveryv1.EndpointSlice, error) ListCACerts() ([]*corev1.Secret, error) @@ -156,6 +157,19 @@ func (s Store) GetSecret(namespace, name string) (*corev1.Secret, error) { return secret.(*corev1.Secret), nil } +// GetConfigMap returns a ConfigMap using the namespace and name as key. +func (s Store) GetConfigMap(namespace, name string) (*corev1.ConfigMap, error) { + key := fmt.Sprintf("%v/%v", namespace, name) + configMap, exists, err := s.stores.ConfigMap.GetByKey(key) + if err != nil { + return nil, err + } + if !exists { + return nil, NotFoundError{fmt.Sprintf("ConfigMap %v not found", key)} + } + return configMap.(*corev1.ConfigMap), nil +} + // GetService returns a Service using the namespace and name as key. func (s Store) GetService(namespace, name string) (*corev1.Service, error) { key := fmt.Sprintf("%v/%v", namespace, name) diff --git a/internal/store/zz_generated.cache_stores.go b/internal/store/zz_generated.cache_stores.go index fd1e3f6b1c..8dfc130154 100644 --- a/internal/store/zz_generated.cache_stores.go +++ b/internal/store/zz_generated.cache_stores.go @@ -27,6 +27,7 @@ type CacheStores struct { IngressClassV1 cache.Store Service cache.Store Secret cache.Store + ConfigMap cache.Store EndpointSlice cache.Store HTTPRoute cache.Store UDPRoute cache.Store @@ -59,6 +60,7 @@ func NewCacheStores() CacheStores { IngressClassV1: cache.NewStore(clusterWideKeyFunc), Service: cache.NewStore(namespacedKeyFunc), Secret: cache.NewStore(namespacedKeyFunc), + ConfigMap: cache.NewStore(namespacedKeyFunc), EndpointSlice: cache.NewStore(namespacedKeyFunc), HTTPRoute: cache.NewStore(namespacedKeyFunc), UDPRoute: cache.NewStore(namespacedKeyFunc), diff --git a/test/integration/isolated/ingress_verify_upstream_tls_test.go b/test/integration/isolated/ingress_verify_upstream_tls_test.go index a79dfd9f50..58ccb11df2 100644 --- a/test/integration/isolated/ingress_verify_upstream_tls_test.go +++ b/test/integration/isolated/ingress_verify_upstream_tls_test.go @@ -195,10 +195,10 @@ func TestIngressVerifyUpstreamTLS(t *testing.T) { t.Logf("Setting up service annotations for TLS verification") service.Annotations = map[string]string{ - annotations.AnnotationPrefix + annotations.ProtocolKey: "https", // Only https or tls are supported. - annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", - annotations.AnnotationPrefix + annotations.CACertificatesKey: strings.Join([]string{caSecretName, anotherCASecretName}, ","), - annotations.AnnotationPrefix + annotations.TLSVerifyDepthKey: "0", // First, we'll set it to 0 to make sure it fails. + annotations.AnnotationPrefix + annotations.ProtocolKey: "https", // Only https or tls are supported. + annotations.AnnotationPrefix + annotations.TLSVerifyKey: "true", + annotations.AnnotationPrefix + annotations.CACertificatesFromSecretKey: strings.Join([]string{caSecretName, anotherCASecretName}, ","), + annotations.AnnotationPrefix + annotations.TLSVerifyDepthKey: "0", // First, we'll set it to 0 to make sure it fails. } service, err = cluster.Client().CoreV1().Services(ns).Create(ctx, service, metav1.CreateOptions{}) assert.NoError(t, err)