adding CNCF runner

Signed-off-by: KMAnju-2021 <[email protected]>
antrea-io · Jan 6, 2025 · 325a3d3 · 325a3d3
1 parent b5f3713
commit 325a3d3
Show file tree

Hide file tree

Showing 4 changed files with 186 additions and 21 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -277,6 +277,39 @@ jobs:
           path: log.tar.gz
           retention-days: 30
 
+  test-e2e-ipam-feature-enabled:
+    name: E2e tests on a Kind cluster on Linux with FlexibleIPAM feature enabled
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest-4-cores]
+    steps:
+      - name: Free disk space
+        # https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo apt-get clean
+          df -h
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - name: Download Antrea image from previous job
+        uses: actions/download-artifact@v4
+        with:
+          name: antrea-ubuntu-cov
+      - name: Load Antrea image
+        run: |
+          docker load -i antrea-ubuntu.tar
+      - name: Install Kind
+        run: |
+          KIND_VERSION=$(head -n1 ./ci/kind/version)
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+      - name: Run ipam e2e tests
+        run: |
+          ./ci/kind/test-e2e-kind.sh --flexible-ipam --encap-mode noEncap
+
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
     needs: [build-antrea-coverage-image]

diff --git a/ci/kind/Dockerfile.ipset b/ci/kind/Dockerfile.ipset
@@ -0,0 +1,62 @@
+# Copyright 2023 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:24.04@sha256:278628f08d4979fb9af9ead44277dbc9c92c2465922310916ad0c46ec9999295
+
+LABEL maintainer="Antrea <[email protected]>"
+LABEL description="A Docker image with useful tools (for networking) based on Ubuntu"
+
+# Keep it in lexicographic order
+ENV TOOLS apache2-utils \
+    arping \
+    conntrack \
+    curl \
+    dnsutils \
+    ethtool \
+    iperf3 \
+    iproute2 \
+    iptables \
+    iputils-ping \
+    ipset\
+    kmod \
+    ncat \
+    netperf \
+    nmap \
+    socat \
+    tcpdump \
+    telnet \
+    traceroute \
+    wget
+
+# See https://github.com/kubernetes-sigs/iptables-wrappers
+ADD --chmod=700 https://raw.githubusercontent.com/kubernetes-sigs/iptables-wrappers/e139a115350974aac8a82ec4b815d2845f86997e/iptables-wrapper-installer.sh /iptables-wrapper-installer.sh
+
+# We run /iptables-wrapper-installer.sh with --no-sanity-check to avoid an issue
+# when building the arm64 version of this docker image with qemu:
+# Failed to initialize nft: Protocol not supported
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $TOOLS && \
+    rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \
+    /iptables-wrapper-installer.sh --no-sanity-check
+
+COPY VERSION /VERSION
+
+# Use K8s pause binary as the default command.
+# This is convenient when using this Docker image to run a K8s Pod which is meant to be exec'ed
+# into, as there is no need to specify an explicit command when deploying the Pod.
+# "pause" is superior to a simple "sleep infinity", as it handled signals correctly.
+# Note that pause is a static binary.
+COPY --from=registry.k8s.io/pause:3.9 /pause /pause
+
+CMD ["/pause"]
diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh
@@ -108,7 +108,16 @@ function add_option {
 }
 
 function docker_run_with_host_net {
-  docker run --rm --net=host --privileged antrea/toolbox:latest "$@"
+  local default_image="antrea/toolbox:latest"
+  if [ "$#" -eq 1]; then
+    local image=$default_image
+    local command="$@"
+  else 
+    local image="$1"
+    shift
+    local command="$@"
+  fi
+  docker run --rm --net=host --privileged $image $command
 }
 
 function configure_networks {
@@ -123,9 +132,9 @@ function configure_networks {
   # Inject allow all iptables to preempt docker bridge isolation rules
   if [[ ! -z $SUBNETS ]]; then
     set +e
-    docker_run_with_host_net iptables -C DOCKER-USER -j ACCEPT > /dev/null 2>&1
+    docker_run_with_host_net "iptables -C DOCKER-USER -j ACCEPT > /dev/null 2>&1"
     if [[ $? -ne 0 ]]; then
-      docker_run_with_host_net iptables -I DOCKER-USER -j ACCEPT
+      docker_run_with_host_net "iptables -I DOCKER-USER -j ACCEPT"
     fi
     set -e
   fi
@@ -257,7 +266,7 @@ function update_kind_ipam_routes {
     return
   fi
   echo "$node_data"| while read pod_cidr node_ip; do
-    docker_run_with_host_net ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true
+    docker_run_with_host_net "ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true"
   done
 }
 
@@ -279,27 +288,47 @@ function configure_vlan_subnets {
     vlan_interface="br-${bridge_id:0:7}.$vlan_id"
     vlan_interfaces+=("$vlan_interface")
 
-    docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id
-    docker_run_with_host_net ip link set $vlan_interface up
+    docker_run_with_host_net "ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id"
+    docker_run_with_host_net "ip link set $vlan_interface up"
 
     IFS=',' read -r -a subnet_array <<< "$subnets"
     for subnet in "${subnet_array[@]}" ; do
       echo "Configuring extra IP $subnet to VLAN interface $vlan_interface"
-      docker_run_with_host_net ip addr add dev $vlan_interface $subnet
+      docker_run_with_host_net "ip addr add dev $vlan_interface $subnet"
     done
 
-    docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT
-    docker_run_with_host_net iptables -t filter -A FORWARD -i $vlan_interface -o $bridge_interface -j ACCEPT
-    docker_run_with_host_net iptables -t filter -A FORWARD -i $vlan_interface -o $vlan_interface -j ACCEPT
+    docker_run_with_host_net "iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT"
+    docker_run_with_host_net "iptables -t filter -A FORWARD -i $vlan_interface -o $bridge_interface -j ACCEPT"
+    docker_run_with_host_net "iptables -t filter -A FORWARD -i $vlan_interface -o $vlan_interface -j ACCEPT"
   done
 
   # Allow traffic between VLANs
   for ((i=0; i<${#vlan_interfaces[@]}; i++)); do
     for ((j=i+1; j<${#vlan_interfaces[@]}; j++)); do
-      docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[i]} -o ${vlan_interfaces[j]} -j ACCEPT
-      docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT
+      docker_run_with_host_net "iptables -t filter -A FORWARD -i ${vlan_interfaces[i]} -o ${vlan_interfaces[j]} -j ACCEPT"
+      docker_run_with_host_net "iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT"
     done
   done
+
+  # Adding ipset rule to prevent SNAT based on flexibleIPAM e2e tests
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+    docker buildx build -t antrea/toolbox:ipset -f ./ci/kind/Dockerfile.ipset .
+    IMAGE="antrea/toolbox:ipset"
+
+    docker_run_with_host_net "$IMAGE" "ipset create excluded_subnets hash:net"
+    docker_run_with_host_net "$IMAGE" "ipset add excluded_subnets 192.168.241.0/24"
+    docker_run_with_host_net "$IMAGE" "ipset add excluded_subnets 192.168.242.0/24"
+    docker_run_with_host_net "$IMAGE" "ipset list excluded_subnets"
+    docker_run_with_host_net "$IMAGE" "iptables -t nat -A POSTROUTING  ! -o  $bridge_interface -s 192.168.240.0/24 -m set ! --match-set excluded_subnets dst -j MASQUERADE"
+
+    docker_run_with_host_net "$IMAGE" "ipset create excluded_ipam_subnets hash:net"
+    docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.241.0/24"
+    docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.242.0/24"
+    docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.240.0/24"
+    docker_run_with_host_net "$IMAGE" "ipset list excluded_ipam_subnets"
+    docker_run_with_host_net "$IMAGE" "iptables -t nat -A POSTROUTING  ! -o  $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_ipam_subnets dst -j MASQUERADE"
+  fi 
+
 }
 
 function delete_vlan_subnets {
@@ -309,13 +338,13 @@ function delete_vlan_subnets {
   bridge_interface="br-${bridge_id:0:12}"
   vlan_interface_prefix="br-${bridge_id:0:7}."
 
-  found_vlan_interfaces=$(docker_run_with_host_net ip -br link show type vlan | cut -d " " -f 1)
+  found_vlan_interfaces=$(docker_run_with_host_net "ip -br link show type vlan | cut -d " " -f 1")
   for interface in $found_vlan_interfaces ; do
     if [[ $interface =~ ${vlan_interface_prefix}[0-9]+@${bridge_interface} ]]; then
       interface_name=${interface%@*}
-      docker_run_with_host_net iptables -t filter -D FORWARD -i $bridge_interface -o $interface_name -j ACCEPT || true
-      docker_run_with_host_net iptables -t filter -D FORWARD -o $bridge_interface -i $interface_name -j ACCEPT || true
-      docker_run_with_host_net ip link del $interface_name
+      docker_run_with_host_net "iptables -t filter -D FORWARD -i $bridge_interface -o $interface_name -j ACCEPT || true"
+      docker_run_with_host_net "iptables -t filter -D FORWARD -o $bridge_interface -i $interface_name -j ACCEPT || true"
+      docker_run_with_host_net "ip link del $interface_name"
     fi
   done
 }
@@ -327,6 +356,13 @@ function delete_networks {
     docker network rm $networks > /dev/null 2>&1
     echo "deleted networks $networks"
   fi
+
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+     networks=$(docker network ls -f name=kind --format '{{.Name}}')
+     networks="$(echo $networks)"
+     docker network rm $networks > /dev/null 2>&1
+     echo "deleted networks $networks"
+  fi 
 }
 
 function load_images {
@@ -711,7 +747,6 @@ if [[ $ACTION == "destroy" ]]; then
       exit
 fi
 
-
 kind_version=$(kind version | awk  '{print $2}')
 kind_version=${kind_version:1} # strip leading 'v'
 function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; }
@@ -728,5 +763,10 @@ if [[ $ACTION == "create" ]]; then
         echoerr "Only one of '--subnets' and '--extra-networks' can be specified"
         exit 1
     fi
+
+    # To run FlexibleIPAM e2e tests, kind network should be in 192.168.240.0/24 subnet.  
+    if [[ $FLEXIBLE_IPAM == true ]]; then
+        docker network create -d bridge --subnet 192.168.240.0/24 kind
+    fi
     create
 fi
diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -89,6 +89,7 @@ setup_only=false
 cleanup_only=false
 test_only=false
 run=""
+flexible_ipam=false
 antrea_controller_image="antrea/antrea-controller-ubuntu"
 antrea_agent_image="antrea/antrea-agent-ubuntu"
 use_non_default_images=false
@@ -110,6 +111,10 @@ case $key in
     proxy_all=true
     shift
     ;;
+    --flexible-ipam)
+    flexible_ipam=true
+    shift
+    ;;
     --no-kube-proxy)
     no_kube_proxy=true
     shift
@@ -249,6 +254,10 @@ if $flow_visibility; then
     manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
 fi
 
+if $flexible_ipam; then
+    manifest_args="$manifest_args --flexible-ipam --multicast"
+fi
+
 COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \
                     "antrea/nginx:1.21.6-alpine" \
                     "antrea/toolbox:1.3-0")
@@ -302,6 +311,10 @@ if $extra_vlan; then
   fi
 fi
 
+if $flexible_ipam; then
+   vlan_args="$vlan_args --vlan-subnets 11=192.168.241.1/24 --vlan-subnets 12=192.168.242.1/24" 
+fi
+
 function setup_cluster {
   args=$1
 
@@ -330,7 +343,11 @@ function setup_cluster {
   fi
 
   echo "creating test bed with args $args"
-  eval "timeout 600 $TESTBED_CMD create kind $args"
+  if $flexible_ipam; then
+     eval "timeout 600 $TESTBED_CMD --flexible-ipam create kind $args" 
+  else
+     eval "timeout 600 $TESTBED_CMD create kind $args"
+  fi
 }
 
 function run_test {
@@ -348,8 +365,13 @@ function run_test {
       timeout="80m"
       coverage_args="--coverage --coverage-dir $ANTREA_COV_DIR"
   else
-      $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
-      $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml
+      if $flexible_ipam; then
+        $YML_CMD --flexible-ipam --multicast --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
+        echo "debug-1"
+      else
+        $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
+        $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml
+      fi
       timeout="75m"
   fi
 
@@ -401,7 +423,15 @@ function run_test {
     EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips"
   fi
 
-  go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS
+  if $flexible_ipam; then
+    sudo iptables -t nat -vnL
+    kubectl get pods -o wide -A
+    ip route
+    export GO111MODULE=on
+    go test -v antrea.io/antrea/test/e2e --provider kind -timeout=100m --prometheus --antrea-ipam
+  else
+     go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS
+  fi
 
   if $coverage; then
     pushd $ANTREA_COV_DIR