Add e2e test for FQDN policy rule enforcement when apps cache DNS (#6695

)

For #6229 

Pending the implementation of the minTTL feature, we add an e2e
test to validate the behavior of FQDN policy rule enforcement when
an application is caching DNS responses beyond the included TTL.

As of now, the Antrea implementation only caches the IP obtained from
the DNS response for the duration specified by the TTL. This means that
if an application keeps using the same IP (because it was cached) beyond
that TTL for a whitelisted FQDN, it will eventually fail. After we add
support for the minTTL configuration parameter, the test will be updated
to validate that the minTTL value is honored, and that until minTTL
"expires", the application can keep using the same IP successfully.

Signed-off-by: Hemant <[email protected]>

test/e2e/antreapolicy_test.go

@@ -15,6 +15,7 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -24,6 +25,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"text/template"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -35,9 +37,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/utils/ptr"
 
 	"antrea.io/antrea/pkg/agent/apis"
 	crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1"
+	agentconfig "antrea.io/antrea/pkg/config/agent"
 	"antrea.io/antrea/pkg/controller/networkpolicy"
 	"antrea.io/antrea/pkg/features"
 	. "antrea.io/antrea/test/e2e/utils"
@@ -5265,3 +5269,252 @@ func testAntreaClusterNetworkPolicyStats(t *testing.T, data *TestData) {
 	}
 	k8sUtils.Cleanup(namespaces)
 }
+
+// TestFQDNCacheMinTTL tests stable FQDN access for applications with cached DNS resolutions
+// when FQDN NetworkPolicy are in use and the FQDN-to-IP resolution changes frequently.
+func TestFQDNCacheMinTTL(t *testing.T) {
+	const (
+		testFQDN = "fqdn-test-pod.lfx.test"
+		dnsPort  = 53
+		dnsTTL   = 5
+	)
+
+	skipIfAntreaPolicyDisabled(t)
+	skipIfNotIPv4Cluster(t)
+	skipIfIPv6Cluster(t)
+	skipIfNotRequired(t, "mode-irrelevant")
+
+	data, err := setupTest(t)
+	if err != nil {
+		t.Fatalf("Error when setting up test: %v", err)
+	}
+	defer teardownTest(t, data)
+
+	// create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN.
+	podCount := 2
+	agnhostPodIPs := make([]*PodIPs, podCount)
+	for i := 0; i < podCount; i++ {
+		agnhostPodIPs[i] = createHttpAgnhostPod(t, data)
+	}
+
+	// get IPv4 addresses of the agnhost Pods created.
+	agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings()
+	agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings()
+
+	// create customDNS Service and get its ClusterIP.
+	customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort,
+		dnsPort, v1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false,
+		false, v1.ServiceTypeClusterIP, ptr.To[v1.IPFamily](v1.IPv4Protocol), map[string]string{})
+	require.NoError(t, err, "Error creating custom DNS Service")
+	dnsServiceIP := customDNSService.Spec.ClusterIP
+
+	// create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN.
+	configMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "custom-dns-config",
+			Namespace: data.testNamespace,
+		},
+		Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL),
+	}
+	customDNSConfigMap, err := data.CreateConfigMap(configMap)
+	require.NoError(t, err, "failed to create custom DNS ConfigMap")
+
+	createCustomDNSPod(t, data, configMap.Name)
+
+	// set the custom DNS server IP address in Antrea ConfigMap.
+	setDNSServerAddressInAntrea(t, data, dnsServiceIP)
+	defer setDNSServerAddressInAntrea(t, data, "") //reset after the test.
+
+	createPolicyForFQDNCacheMinTTL(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test")
+	require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage).
+		WithLabels(map[string]string{"app": "fqdn-cache-test"}).
+		WithContainerName(toolboxContainerName).
+		WithCustomDNSConfig(&v1.PodDNSConfig{Nameservers: []string{dnsServiceIP}}).
+		Create(data))
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace))
+
+	curlFQDN := func(target string) (string, error) {
+		cmd := []string{"curl", target}
+		stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd)
+		if err != nil {
+			return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>",
+				strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr)
+		}
+		return stdout, nil
+	}
+
+	assert.EventuallyWithT(t, func(t *assert.CollectT) {
+		_, err := curlFQDN(testFQDN)
+		assert.NoError(t, err)
+	}, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN)
+
+	// confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod.
+	t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod")
+	resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP)
+	fqdnIP := resolvedIP.String()
+	require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod")
+	require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value")
+	t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP)
+
+	// update the IP address mapped to the FQDN in the custom DNS ConfigMap.
+	t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP)
+	customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL)
+	require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP")
+	t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP)
+
+	// try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way
+	// we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps.
+	// Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically
+	require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value",
+		randSeq(8)), "failed to update custom DNS Pod annotation.")
+
+	// finally verify that Curling the previously cached IP fails after DNS update.
+	// The wait time here should be slightly longer than the reload value specified in the custom DNS configuration.
+	// TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229.
+	// It will need to be updated once minTTL support is implemented.
+	t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP)
+	assert.EventuallyWithT(t, func(t *assert.CollectT) {
+		_, err := curlFQDN(fqdnIP)
+		assert.Error(t, err)
+	}, 10*time.Second, 1*time.Second)
+}
+
+// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap.
+func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) {
+	agentChanges := func(config *agentconfig.AgentConfig) {
+		config.DNSServerOverride = dnsServiceIP
+	}
+	err := data.mutateAntreaConfigMap(nil, agentChanges, false, true)
+	require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap")
+
+	t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP)
+}
+
+// createPolicyForFQDNCacheMinTTL creates a FQDN policy in the specified Namespace.
+func createPolicyForFQDNCacheMinTTL(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) {
+	podSelectorLabel := map[string]string{
+		"app": fqdnPodSelectorLabelValue,
+	}
+	builder := &AntreaNetworkPolicySpecBuilder{}
+	builder = builder.SetName(data.testNamespace, fqdnPolicyName).
+		SetTier(defaultTierName).
+		SetPriority(1.0).
+		SetAppliedToGroup([]ANNPAppliedToSpec{{PodSelector: podSelectorLabel}})
+	builder.AddFQDNRule(testFQDN, ProtocolTCP, ptr.To[int32](80), nil, nil, "AllowForFQDN", nil,
+		crdv1beta1.RuleActionAllow)
+	builder.AddEgress(ProtocolUDP, ptr.To[int32](53), nil, nil, nil, nil,
+		nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue},
+		nil, nil, nil, nil,
+		nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries")
+	builder.AddEgress(ProtocolTCP, nil, nil, nil, nil, nil,
+		nil, nil, nil, nil, nil,
+		nil, nil, nil, nil,
+		nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic")
+
+	annp, err := data.CreateOrUpdateANNP(builder.Get())
+	require.NoError(t, err, "error while deploying Antrea policy")
+	require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second))
+}
+
+// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created.
+func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs {
+	const (
+		agnhostPort          = 80
+		agnhostPodNamePreFix = "agnhost-"
+	)
+	podName := randName(agnhostPodNamePreFix)
+	args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)}
+	ports := []v1.ContainerPort{
+		{
+			Name:          "http",
+			ContainerPort: agnhostPort,
+			Protocol:      v1.ProtocolTCP,
+		},
+	}
+
+	require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).
+		WithArgs(args).
+		WithPorts(ports).
+		WithLabels(map[string]string{"app": "agnhost"}).
+		Create(data))
+	podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace)
+	require.NoError(t, err)
+	return podIPs
+}
+
+// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap.
+func createCustomDNSPod(t *testing.T, data *TestData, configName string) {
+	volume := []v1.Volume{
+		{
+			Name: "config-volume",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: configName,
+					},
+					Items: []v1.KeyToPath{
+						{
+							Key:  "Corefile",
+							Path: "Corefile",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	volumeMount := []v1.VolumeMount{
+		{
+			Name:      "config-volume",
+			MountPath: "/etc/coredns",
+		},
+	}
+
+	require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3").
+		WithLabels(map[string]string{"app": "custom-dns"}).
+		WithContainerName("coredns").
+		WithArgs([]string{"-conf", "/etc/coredns/Corefile"}).
+		AddVolume(volume).AddVolumeMount(volumeMount).
+		Create(data))
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace))
+}
+
+// createDNSConfig generates a DNS configuration for the specified IP address and domain name.
+func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string {
+	const coreFileTemplate = `lfx.test:53 {
+        errors
+        log
+        health
+        hosts {
+            {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }}
+            no_reverse
+            pods verified
+            ttl {{ .TTL }}
+        }
+        loop
+        reload 2s
+    }`
+
+	data := struct {
+		Hosts map[string]string
+		TTL   int
+	}{
+		Hosts: hosts,
+		TTL:   ttl,
+	}
+
+	// Parse the template and generate the config data
+	tmpl, err := template.New("configMapData").Parse(coreFileTemplate)
+	require.NoError(t, err, "error parsing config template")
+
+	var output bytes.Buffer
+	err = tmpl.Execute(&output, data)
+	require.NoError(t, err, "error executing config template")
+
+	configMapData := strings.TrimSpace(output.String())
+	configData := map[string]string{
+		"Corefile": configMapData,
+	}
+
+	return configData
+}

test/e2e/framework.go

@@ -331,6 +331,16 @@ func (p *PodIPs) AsSlice() []*net.IP {
 	return ips
 }
 
+func (p *PodIPs) AsStrings() (ipv4, ipv6 string) {
+	if p.IPv4 != nil {
+		ipv4 = p.IPv4.String()
+	}
+	if p.IPv6 != nil {
+		ipv6 = p.IPv6.String()
+	}
+	return
+}
+
 // workerNodeName returns an empty string if there is no worker Node with the provided idx
 // (including if idx is 0, which is reserved for the control-plane Node)
 func workerNodeName(idx int) string {
@@ -1357,6 +1367,7 @@ type PodBuilder struct {
 	ResourceRequests   corev1.ResourceList
 	ResourceLimits     corev1.ResourceList
 	ReadinessProbe     *corev1.Probe
+	DnsConfig          *corev1.PodDNSConfig
 }
 
 func NewPodBuilder(name, ns, image string) *PodBuilder {
@@ -1481,6 +1492,13 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder {
 	return b
 }
 
+// WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec.
+// It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig.
+func (b *PodBuilder) WithCustomDNSConfig(dnsConfig *corev1.PodDNSConfig) *PodBuilder {
+	b.DnsConfig = dnsConfig
+	return b
+}
+
 func (b *PodBuilder) Create(data *TestData) error {
 	containerName := b.ContainerName
 	if containerName == "" {
@@ -1523,6 +1541,13 @@ func (b *PodBuilder) Create(data *TestData) error {
 		// tolerate NoSchedule taint if we want Pod to run on control-plane Node
 		podSpec.Tolerations = controlPlaneNoScheduleTolerations()
 	}
+	if b.DnsConfig != nil {
+		// Set DNSPolicy to None to allow custom DNSConfig
+		podSpec.DNSPolicy = corev1.DNSNone
+
+		// Assign the provided DNSConfig to the Pod's DNSConfig field
+		podSpec.DNSConfig = b.DnsConfig
+	}
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        b.Name,
@@ -3220,3 +3245,54 @@ func (data *TestData) GetPodLogs(ctx context.Context, namespace, name, container
 	}
 	return b.String(), nil
 }
+
+func (data *TestData) runDNSQuery(
+	podName string,
+	containerName string,
+	podNamespace string,
+	dstAddr string,
+	useTCP bool,
+	dnsServiceIP string) (net.IP, error) {
+
+	digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr)
+	if useTCP {
+		digCmdStr += " +tcp"
+	}
+
+	digCmd := strings.Fields(digCmdStr)
+	stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd)
+	if err != nil {
+		return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr)
+	}
+
+	ipAddress := net.ParseIP(strings.TrimSpace(stdout))
+	if ipAddress != nil {
+		return ipAddress, nil
+	} else {
+		return nil, fmt.Errorf("invalid IP address found %v", stdout)
+	}
+}
+
+// setPodAnnotation Patches a pod by adding an annotation with a specified key and value.
+func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error {
+	annotations := map[string]string{
+		annotationKey: annotationValue,
+	}
+	annotationPatch := map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"annotations": annotations,
+		},
+	}
+
+	patchData, err := json.Marshal(annotationPatch)
+	if err != nil {
+		return err
+	}
+
+	if _, err := data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}); err != nil {
+		return err
+	}
+
+	log.Infof("Successfully patched Pod %s in Namespace %s", podName, namespace)
+	return nil
+}