feat: convert BackendTLSPolicies into service annotations

CHANGELOG.md

@@ -228,7 +228,16 @@ Adding a new version? You'll need three changes:
   By setting this flag, the secrets that are ingested will be limited to those having this label set to "true".
   This can reduce the memory usage in scenarios with a large number of giant secrets.
   [#6795](https://github.com/Kong/kubernetes-ingress-controller/pull/6795)
-
+- Added `BackendTLSPolicy` support. The user can now reference any Kubernetes `Service`
+  in the `BackendTLSPolicy` spec, and in case the service is used as a backend by
+  `HTTPRoute`s that reference a Kong Gateway as parent, such Backend TLS configuration
+  is applied to the service section of the Kong configuration.
+  [#6712](https://github.com/Kong/kubernetes-ingress-controller/pull/6712)
+  [#6753](https://github.com/Kong/kubernetes-ingress-controller/pull/6753)
+- Added the flag `--configmap-label-selector` to set the label selector for `ConfigMap`s
+  to ingest. By setting this flag, the `ConfigMap`s that are ingested will be limited
+  to those having this label set to "true". This limits the amount of resources that are kept in memory. The default value is `konghq.com/configmap`.
+  [#6753](https://github.com/Kong/kubernetes-ingress-controller/pull/6753)
 
 ## [3.3.1]
 

config/rbac/gateway/role.yaml

@@ -7,6 +7,7 @@ rules:
 - apiGroups:
   - ""
   resources:
+  - configmaps
   - namespaces
   - services
   verbs:

config/rbac/role.yaml

@@ -7,18 +7,19 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - events
-  verbs:
-  - create
-  - patch
-- apiGroups:
-  - ""
-  resources:
+  - configmaps
   - nodes
   - secrets
   verbs:
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - ""
   resources:

docs/cli-arguments.md

@@ -15,6 +15,7 @@
 | `--apiserver-qps` | `int` | The Kubernetes API RateLimiter maximum queries per second. | `100` |
 | `--cache-sync-timeout` | `duration` | The time limit set to wait for syncing controllers' caches. Set to 0 to use default from controller-runtime. | `2m0s` |
 | `--cluster-domain` | `string` | The cluster domain. This is used e.g. in generating addresses for upstream services. |  |
+| `--configmap-label-selector` | `string` | Limits the configmaps ingested to those having this label set to "true". | `konghq.com/configmap` |
 | `--dump-config` | `bool` | Enable config dumps via web interface host:10256/debug/config. | `false` |
 | `--dump-sensitive-config` | `bool` | Include credentials and TLS secrets in configs exposed with --dump-config flag. | `false` |
 | `--election-id` | `string` | Election id to use for status update. | `5b374a9e.konghq.com` |

examples/ingress-upstream-tls.yaml

@@ -42,10 +42,10 @@ metadata:
     app: goecho
   name: goecho
   annotations:
-    konghq.com/tls-verify: "true"    # Enable TLS verification of the upstream.
-    konghq.com/ca-certificates: "ca" # The CA root certificate secret used for verification.
-    konghq.com/protocol: "https"     # Has to be either https or tls when TLS verification is enabled.
-    konghq.com/host-header: "goecho" # This will make Kong use `goecho` server name when validating server-presented TLS certificate.
+    konghq.com/tls-verify: "true"           # Enable TLS verification of the upstream.
+    konghq.com/ca-certificates-secret: "ca" # The CA root certificate secret used for verification.
+    konghq.com/protocol: "https"            # Has to be either https or tls when TLS verification is enabled.
+    konghq.com/host-header: "goecho"        # This will make Kong use `goecho` server name when validating server-presented TLS certificate.
 spec:
   ports:
   - port: 443

hack/generators/cache-stores/spec.go

@@ -23,6 +23,10 @@ var supportedTypes = []cacheStoreSupportedType{
 		Type:    "Secret",
 		Package: "corev1",
 	},
+	{
+		Type:    "ConfigMap",
+		Package: "corev1",
+	},
 	{
 		Type:    "EndpointSlice",
 		Package: "discoveryv1",

internal/annotations/annotations.go

@@ -41,35 +41,36 @@ const (
 
 	AnnotationPrefix = "konghq.com"
 
-	ConfigurationKey     = "/override"
-	PluginsKey           = "/plugins"
-	ProtocolKey          = "/protocol"
-	ProtocolsKey         = "/protocols"
-	ClientCertKey        = "/client-cert"
-	StripPathKey         = "/strip-path"
-	PathKey              = "/path"
-	HTTPSRedirectCodeKey = "/https-redirect-status-code"
-	PreserveHostKey      = "/preserve-host"
-	RegexPriorityKey     = "/regex-priority"
-	HostHeaderKey        = "/host-header"
-	MethodsKey           = "/methods"
-	SNIsKey              = "/snis"
-	RequestBuffering     = "/request-buffering"
-	ResponseBuffering    = "/response-buffering"
-	HostAliasesKey       = "/host-aliases"
-	RegexPrefixKey       = "/regex-prefix"
-	ConnectTimeoutKey    = "/connect-timeout"
-	WriteTimeoutKey      = "/write-timeout"
-	ReadTimeoutKey       = "/read-timeout"
-	RetriesKey           = "/retries"
-	HeadersKey           = "/headers"
-	HeadersSeparatorKey  = "/headers-separator"
-	PathHandlingKey      = "/path-handling"
-	UserTagKey           = "/tags"
-	RewriteURIKey        = "/rewrite"
-	TLSVerifyKey         = "/tls-verify"
-	TLSVerifyDepthKey    = "/tls-verify-depth"
-	CACertificatesKey    = "/ca-certificates"
+	ConfigurationKey            = "/override"
+	PluginsKey                  = "/plugins"
+	ProtocolKey                 = "/protocol"
+	ProtocolsKey                = "/protocols"
+	ClientCertKey               = "/client-cert"
+	StripPathKey                = "/strip-path"
+	PathKey                     = "/path"
+	HTTPSRedirectCodeKey        = "/https-redirect-status-code"
+	PreserveHostKey             = "/preserve-host"
+	RegexPriorityKey            = "/regex-priority"
+	HostHeaderKey               = "/host-header"
+	MethodsKey                  = "/methods"
+	SNIsKey                     = "/snis"
+	RequestBuffering            = "/request-buffering"
+	ResponseBuffering           = "/response-buffering"
+	HostAliasesKey              = "/host-aliases"
+	RegexPrefixKey              = "/regex-prefix"
+	ConnectTimeoutKey           = "/connect-timeout"
+	WriteTimeoutKey             = "/write-timeout"
+	ReadTimeoutKey              = "/read-timeout"
+	RetriesKey                  = "/retries"
+	HeadersKey                  = "/headers"
+	HeadersSeparatorKey         = "/headers-separator"
+	PathHandlingKey             = "/path-handling"
+	UserTagKey                  = "/tags"
+	RewriteURIKey               = "/rewrite"
+	TLSVerifyKey                = "/tls-verify"
+	TLSVerifyDepthKey           = "/tls-verify-depth"
+	CACertificatesSecretsKey    = "/ca-certificates-secret"
+	CACertificatesConfigMapsKey = "/ca-certificates-configmap"
 
 	// GatewayClassUnmanagedKey is an annotation used on a Gateway resource to
 	// indicate that the GatewayClass should be reconciled according to unmanaged
@@ -395,10 +396,18 @@ func ExtractTLSVerifyDepth(anns map[string]string) (int, bool) {
 	return depth, true
 }
 
-// ExtractCACertificates extracts the ca-certificates secret names from the annotation.
-// It expects a comma-separated list of certificate names.
-func ExtractCACertificates(anns map[string]string) []string {
-	s, ok := anns[AnnotationPrefix+CACertificatesKey]
+// ExtractCACertificateSecretNames extracts the ca-certificates secret names from the `ca-certificates-secret` annotation.
+// It expects a comma-separated list of secret names containing CA certificates.
+func ExtractCACertificateSecretNames(anns map[string]string) []string {
+	s, ok := anns[AnnotationPrefix+CACertificatesSecretsKey]
+	if !ok {
+		return nil
+	}
+	return extractCommaDelimitedStrings(s)
+}
+
+func ExtractCACertificateConfigMapNames(anns map[string]string) []string {
+	s, ok := anns[AnnotationPrefix+CACertificatesConfigMapsKey]
 	if !ok {
 		return nil
 	}
@@ -440,3 +449,33 @@ func extractCommaDelimitedStrings(s string, sanitizeFns ...func(string) string)
 
 	return out
 }
+
+// SetTLSVerify sets the tls-verify annotation value.
+func SetTLSVerify(anns map[string]string, value bool) {
+	anns[AnnotationPrefix+TLSVerifyKey] = strconv.FormatBool(value)
+}
+
+// SetCACertificates merge the ca-certificates secret names into the already existing CA certificates set via annotation.
+func SetCACertificates(anns map[string]string, certificates []string) {
+	existingCACerts := anns[AnnotationPrefix+CACertificatesConfigMapsKey]
+	if existingCACerts == "" {
+		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = strings.Join(certificates, ",")
+	} else {
+		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = existingCACerts + "," + strings.Join(certificates, ",")
+	}
+}
+
+// SetHostHeader sets the host-header annotation value.
+func SetHostHeader(anns map[string]string, value string) {
+	anns[AnnotationPrefix+HostHeaderKey] = value
+}
+
+// SetProtocol sets the protocol annotation value.
+func SetProtocol(anns map[string]string, value string) {
+	anns[AnnotationPrefix+ProtocolKey] = value
+}
+
+// SetTLSVerifyDepth sets the tls-verify-depth annotation value.
+func SetTLSVerifyDepth(anns map[string]string, depth int) {
+	anns[AnnotationPrefix+TLSVerifyDepthKey] = strconv.Itoa(depth)
+}

internal/annotations/annotations_test.go

@@ -1106,19 +1106,19 @@ func TestExtractTLSVerifyDepth(t *testing.T) {
 }
 
 func TestExtractCACertificates(t *testing.T) {
-	v := ExtractCACertificates(nil)
+	v := ExtractCACertificateSecretNames(nil)
 	assert.Empty(t, v)
 
-	v = ExtractCACertificates(map[string]string{})
+	v = ExtractCACertificateSecretNames(map[string]string{})
 	assert.Empty(t, v)
 
-	v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: "foo,bar"})
+	v = ExtractCACertificateSecretNames(map[string]string{AnnotationPrefix + CACertificatesSecretsKey: "foo,bar"})
 	assert.Equal(t, []string{"foo", "bar"}, v, "expected to split by comma")
 
-	v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: " foo, bar ,baz "})
+	v = ExtractCACertificateSecretNames(map[string]string{AnnotationPrefix + CACertificatesSecretsKey: " foo, bar ,baz "})
 	assert.Equal(t, []string{"foo", "bar", "baz"}, v, "expected to trim spaces")
 
-	v = ExtractCACertificates(map[string]string{AnnotationPrefix + CACertificatesKey: "foo, bar,  "})
+	v = ExtractCACertificateSecretNames(map[string]string{AnnotationPrefix + CACertificatesSecretsKey: "foo, bar,  "})
 	assert.Equal(t, []string{"foo", "bar"}, v, "expected to ignore empty values")
 }
 

internal/controllers/configuration/configmap_controller.go

@@ -0,0 +1,162 @@
+package configuration
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/controllers"
+	ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/labels"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/logging"
+)
+
+// -----------------------------------------------------------------------------
+// CoreV1 ConfigMap - Reconciler
+// -----------------------------------------------------------------------------
+
+// CoreV1ConfigMapReconciler reconciles ConfigMap resources.
+type CoreV1ConfigMapReconciler struct {
+	client.Client
+
+	Log              logr.Logger
+	Scheme           *runtime.Scheme
+	DataplaneClient  controllers.DataPlane
+	CacheSyncTimeout time.Duration
+
+	ReferenceIndexers ctrlref.CacheIndexers
+	LabelSelector     string
+}
+
+var _ controllers.Reconciler = &CoreV1ConfigMapReconciler{}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *CoreV1ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	predicateFuncs := predicate.NewPredicateFuncs(r.shouldReconcileConfigMap)
+	// we should always try to delete configmaps in caches when they are deleted in cluster.
+	predicateFuncs.DeleteFunc = func(_ event.DeleteEvent) bool { return true }
+
+	var (
+		labelPredicate predicate.Predicate
+		labelSelector  metav1.LabelSelector
+		err            error
+	)
+	if r.LabelSelector != "" {
+		labelSelector = metav1.LabelSelector{
+			MatchLabels: map[string]string{r.LabelSelector: "true"},
+		}
+	}
+
+	labelPredicate, err = predicate.LabelSelectorPredicate(labelSelector)
+	if err != nil {
+		return fmt.Errorf("failed to create secret label selector predicate: %w", err)
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("CoreV1ConfigMap").
+		WithOptions(controller.Options{
+			LogConstructor: func(_ *reconcile.Request) logr.Logger {
+				return r.Log
+			},
+			CacheSyncTimeout: r.CacheSyncTimeout,
+		}).
+		For(&corev1.ConfigMap{},
+			builder.WithPredicates(
+				predicate.And(
+					predicateFuncs,
+					labelPredicate,
+				)),
+		).
+		Complete(r)
+}
+
+// SetLogger sets the logger.
+func (r *CoreV1ConfigMapReconciler) SetLogger(l logr.Logger) {
+	r.Log = l
+}
+
+// shouldReconcileConfigMap is the filter function to judge whether the ConfigMap should be reconciled
+// and stored in cache of the controller. It returns true for the ConfigMap should be reconciled when
+// the ConfigMap is referred by objects we care (BackendTLSPolicy).
+func (r *CoreV1ConfigMapReconciler) shouldReconcileConfigMap(obj client.Object) bool {
+	configMap, ok := obj.(*corev1.ConfigMap)
+	if !ok {
+		return false
+	}
+
+	l := configMap.Labels
+	if l != nil {
+		if l[CACertLabelKey] == "true" {
+			return true
+		}
+
+		if _, ok := l[labels.CredentialTypeLabel]; ok {
+			return true
+		}
+	}
+
+	referred, err := r.ReferenceIndexers.ObjectReferred(configMap)
+	if err != nil {
+		r.Log.Error(err, "Failed to check whether configmap referred",
+			"namespace", configMap.Namespace, "name", configMap.Name)
+		return false
+	}
+
+	return referred
+}
+
+// +kubebuilder:rbac:groups="",resources=configmaps,verbs=list;watch
+
+// Reconcile processes the watched objects.
+func (r *CoreV1ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("CoreV1ConfigMap", req.NamespacedName)
+
+	// get the relevant object
+	configMap := new(corev1.ConfigMap)
+	if err := r.Get(ctx, req.NamespacedName, configMap); err != nil {
+		if apierrors.IsNotFound(err) {
+			configMap.Namespace = req.Namespace
+			configMap.Name = req.Name
+			return ctrl.Result{}, r.DataplaneClient.DeleteObject(configMap)
+		}
+		return ctrl.Result{}, err
+	}
+
+	log.V(logging.DebugLevel).Info("Reconciling ConfigMap resource", "namespace", req.Namespace, "name", req.Name)
+
+	// clean the object up if it's being deleted
+	if !configMap.DeletionTimestamp.IsZero() && time.Now().After(configMap.DeletionTimestamp.Time) {
+		log.V(logging.DebugLevel).Info("Resource is being deleted, its configuration will be removed", "type", "ConfigMap", "namespace", req.Namespace, "name", req.Name)
+		objectExistsInCache, err := r.DataplaneClient.ObjectExists(configMap)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		if objectExistsInCache {
+			if err := r.DataplaneClient.DeleteObject(configMap); err != nil {
+				return ctrl.Result{}, err
+			}
+			return ctrl.Result{Requeue: true}, nil // wait until the object is no longer present in the cache
+		}
+		return ctrl.Result{}, nil
+	}
+
+	// update the kong Admin API with the changes
+	if err := r.DataplaneClient.UpdateObject(configMap); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, nil
+}