diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 0b2ff5efe36..7f7c44829bb 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -15,6 +15,7 @@ package e2e import ( + "bytes" "context" "encoding/json" "fmt" @@ -24,6 +25,7 @@ import ( "strings" "sync" "testing" + "text/template" "time" log "github.com/sirupsen/logrus" @@ -35,9 +37,11 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" "antrea.io/antrea/pkg/agent/apis" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/controller/networkpolicy" "antrea.io/antrea/pkg/features" . "antrea.io/antrea/test/e2e/utils" @@ -5265,3 +5269,252 @@ func testAntreaClusterNetworkPolicyStats(t *testing.T, data *TestData) { } k8sUtils.Cleanup(namespaces) } + +// TestFQDNCacheMinTTL tests stable FQDN access for applications with cached DNS resolutions +// when FQDN NetworkPolicy are in use and the FQDN-to-IP resolution changes frequently. +func TestFQDNCacheMinTTL(t *testing.T) { + const ( + testFQDN = "fqdn-test-pod.lfx.test" + dnsPort = 53 + dnsTTL = 5 + ) + + skipIfAntreaPolicyDisabled(t) + skipIfNotIPv4Cluster(t) + skipIfIPv6Cluster(t) + skipIfNotRequired(t, "mode-irrelevant") + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + // create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. + podCount := 2 + agnhostPodIPs := make([]*PodIPs, podCount) + for i := 0; i < podCount; i++ { + agnhostPodIPs[i] = createHttpAgnhostPod(t, data) + } + + // get IPv4 addresses of the agnhost Pods created. + agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings() + agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings() + + // create customDNS Service and get its ClusterIP. + customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + dnsPort, v1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, + false, v1.ServiceTypeClusterIP, ptr.To[v1.IPFamily](v1.IPv4Protocol), map[string]string{}) + require.NoError(t, err, "Error creating custom DNS Service") + dnsServiceIP := customDNSService.Spec.ClusterIP + + // create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN. + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-dns-config", + Namespace: data.testNamespace, + }, + Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL), + } + customDNSConfigMap, err := data.CreateConfigMap(configMap) + require.NoError(t, err, "failed to create custom DNS ConfigMap") + + createCustomDNSPod(t, data, configMap.Name) + + // set the custom DNS server IP address in Antrea ConfigMap. + setDNSServerAddressInAntrea(t, data, dnsServiceIP) + defer setDNSServerAddressInAntrea(t, data, "") //reset after the test. + + createPolicyForFQDNCacheMinTTL(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). + WithLabels(map[string]string{"app": "fqdn-cache-test"}). + WithContainerName(toolboxContainerName). + WithCustomDNSConfig(&v1.PodDNSConfig{Nameservers: []string{dnsServiceIP}}). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) + + curlFQDN := func(target string) (string, error) { + cmd := []string{"curl", target} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd) + if err != nil { + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr) + } + return stdout, nil + } + + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(testFQDN) + assert.NoError(t, err) + }, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN) + + // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod. + t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod") + resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) + fqdnIP := resolvedIP.String() + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") + require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value") + t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP) + + // update the IP address mapped to the FQDN in the custom DNS ConfigMap. + t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP) + customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL) + require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP") + t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP) + + // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way + // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. + // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically + require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", + randSeq(8)), "failed to update custom DNS Pod annotation.") + + // finally verify that Curling the previously cached IP fails after DNS update. + // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. + // TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229. + // It will need to be updated once minTTL support is implemented. + t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(fqdnIP) + assert.Error(t, err) + }, 10*time.Second, 1*time.Second) +} + +// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap. +func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = dnsServiceIP + } + err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) + require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") + + t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP) +} + +// createPolicyForFQDNCacheMinTTL creates a FQDN policy in the specified Namespace. +func createPolicyForFQDNCacheMinTTL(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) { + podSelectorLabel := map[string]string{ + "app": fqdnPodSelectorLabelValue, + } + builder := &AntreaNetworkPolicySpecBuilder{} + builder = builder.SetName(data.testNamespace, fqdnPolicyName). + SetTier(defaultTierName). + SetPriority(1.0). + SetAppliedToGroup([]ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) + builder.AddFQDNRule(testFQDN, ProtocolTCP, ptr.To[int32](80), nil, nil, "AllowForFQDN", nil, + crdv1beta1.RuleActionAllow) + builder.AddEgress(ProtocolUDP, ptr.To[int32](53), nil, nil, nil, nil, + nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue}, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") + builder.AddEgress(ProtocolTCP, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") + + annp, err := data.CreateOrUpdateANNP(builder.Get()) + require.NoError(t, err, "error while deploying Antrea policy") + require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) +} + +// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created. +func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs { + const ( + agnhostPort = 80 + agnhostPodNamePreFix = "agnhost-" + ) + podName := randName(agnhostPodNamePreFix) + args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)} + ports := []v1.ContainerPort{ + { + Name: "http", + ContainerPort: agnhostPort, + Protocol: v1.ProtocolTCP, + }, + } + + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). + WithArgs(args). + WithPorts(ports). + WithLabels(map[string]string{"app": "agnhost"}). + Create(data)) + podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + require.NoError(t, err) + return podIPs +} + +// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. +func createCustomDNSPod(t *testing.T, data *TestData, configName string) { + volume := []v1.Volume{ + { + Name: "config-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configName, + }, + Items: []v1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + } + + volumeMount := []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + } + + require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3"). + WithLabels(map[string]string{"app": "custom-dns"}). + WithContainerName("coredns"). + WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). + AddVolume(volume).AddVolumeMount(volumeMount). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) +} + +// createDNSConfig generates a DNS configuration for the specified IP address and domain name. +func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }} + no_reverse + pods verified + ttl {{ .TTL }} + } + loop + reload 2s + }` + + data := struct { + Hosts map[string]string + TTL int + }{ + Hosts: hosts, + TTL: ttl, + } + + // Parse the template and generate the config data + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + require.NoError(t, err, "error parsing config template") + + var output bytes.Buffer + err = tmpl.Execute(&output, data) + require.NoError(t, err, "error executing config template") + + configMapData := strings.TrimSpace(output.String()) + configData := map[string]string{ + "Corefile": configMapData, + } + + return configData +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 639beb1e8b1..38901711ff6 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -331,6 +331,16 @@ func (p *PodIPs) AsSlice() []*net.IP { return ips } +func (p *PodIPs) AsStrings() (ipv4, ipv6 string) { + if p.IPv4 != nil { + ipv4 = p.IPv4.String() + } + if p.IPv6 != nil { + ipv6 = p.IPv6.String() + } + return +} + // workerNodeName returns an empty string if there is no worker Node with the provided idx // (including if idx is 0, which is reserved for the control-plane Node) func workerNodeName(idx int) string { @@ -1357,6 +1367,7 @@ type PodBuilder struct { ResourceRequests corev1.ResourceList ResourceLimits corev1.ResourceList ReadinessProbe *corev1.Probe + DnsConfig *corev1.PodDNSConfig } func NewPodBuilder(name, ns, image string) *PodBuilder { @@ -1481,6 +1492,13 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder { return b } +// WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec. +// It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig. +func (b *PodBuilder) WithCustomDNSConfig(dnsConfig *corev1.PodDNSConfig) *PodBuilder { + b.DnsConfig = dnsConfig + return b +} + func (b *PodBuilder) Create(data *TestData) error { containerName := b.ContainerName if containerName == "" { @@ -1523,6 +1541,13 @@ func (b *PodBuilder) Create(data *TestData) error { // tolerate NoSchedule taint if we want Pod to run on control-plane Node podSpec.Tolerations = controlPlaneNoScheduleTolerations() } + if b.DnsConfig != nil { + // Set DNSPolicy to None to allow custom DNSConfig + podSpec.DNSPolicy = corev1.DNSNone + + // Assign the provided DNSConfig to the Pod's DNSConfig field + podSpec.DNSConfig = b.DnsConfig + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: b.Name, @@ -3220,3 +3245,54 @@ func (data *TestData) GetPodLogs(ctx context.Context, namespace, name, container } return b.String(), nil } + +func (data *TestData) runDNSQuery( + podName string, + containerName string, + podNamespace string, + dstAddr string, + useTCP bool, + dnsServiceIP string) (net.IP, error) { + + digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) + if useTCP { + digCmdStr += " +tcp" + } + + digCmd := strings.Fields(digCmdStr) + stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd) + if err != nil { + return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) + } + + ipAddress := net.ParseIP(strings.TrimSpace(stdout)) + if ipAddress != nil { + return ipAddress, nil + } else { + return nil, fmt.Errorf("invalid IP address found %v", stdout) + } +} + +// setPodAnnotation Patches a pod by adding an annotation with a specified key and value. +func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error { + annotations := map[string]string{ + annotationKey: annotationValue, + } + annotationPatch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": annotations, + }, + } + + patchData, err := json.Marshal(annotationPatch) + if err != nil { + return err + } + + if _, err := data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}); err != nil { + return err + } + + log.Infof("Successfully patched Pod %s in Namespace %s", podName, namespace) + return nil +} diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index d665cce4c82..6fe96c27801 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -667,6 +667,14 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } +func (data *TestData) CreateConfigMap(configMap *v1.ConfigMap) (*v1.ConfigMap, error) { + configMapObject, err := data.clientset.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return configMapObject, nil +} + // DeleteService is a convenience function for deleting a Service by Namespace and name. func (data *TestData) DeleteService(ns, name string) error { log.Infof("Deleting Service %s in ns %s", name, ns) diff --git a/test/e2e/utils/annp_spec_builder.go b/test/e2e/utils/annp_spec_builder.go index 670aa584ca4..f3eb1a1b651 100644 --- a/test/e2e/utils/annp_spec_builder.go +++ b/test/e2e/utils/annp_spec_builder.go @@ -214,3 +214,29 @@ func (b *AntreaNetworkPolicySpecBuilder) AddEgressLogging(logLabel string) *Antr } return b } + +func (b *AntreaNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, + protoc AntreaPolicyProtocol, port *int32, portName *string, endPort *int32, name string, + specs []ANNPAppliedToSpec, action crdv1beta1.RuleAction) *AntreaNetworkPolicySpecBuilder { + var appliedTos []crdv1beta1.AppliedTo + + for _, at := range specs { + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, + at.PodSelectorMatchExp, + at.ExternalEntitySelector, + at.ExternalEntitySelectorMatchExp, + at.Group)) + } + + policyPeer := []crdv1beta1.NetworkPolicyPeer{{FQDN: fqdn}} + ports, _ := GenPortsOrProtocols(protoc, port, portName, endPort, nil, nil, nil, nil, nil, nil) + newRule := crdv1beta1.Rule{ + To: policyPeer, + Ports: ports, + Action: &action, + Name: name, + AppliedTo: appliedTos, + } + b.Spec.Egress = append(b.Spec.Egress, newRule) + return b +}