diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 168bf029b33..57a7d2d7655 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -277,6 +277,39 @@ jobs: path: log.tar.gz retention-days: 30 + test-e2e-ipam-feature-enabled: + name: E2e tests on a Kind cluster on Linux with FlexibleIPAM feature enabled + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest-4-cores] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Download Antrea image from previous job + uses: actions/download-artifact@v4 + with: + name: antrea-ubuntu-cov + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu.tar + - name: Install Kind + run: | + KIND_VERSION=$(head -n1 ./ci/kind/version) + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run ipam e2e tests + run: | + ./ci/kind/test-e2e-kind.sh --flexible-ipam --encap-mode noEncap + test-e2e-noencap: name: E2e tests on a Kind cluster on Linux (noEncap) needs: [build-antrea-coverage-image] diff --git a/ci/kind/Dockerfile.ipset b/ci/kind/Dockerfile.ipset new file mode 100644 index 00000000000..062224981bf --- /dev/null +++ b/ci/kind/Dockerfile.ipset @@ -0,0 +1,62 @@ +# Copyright 2023 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:24.04@sha256:278628f08d4979fb9af9ead44277dbc9c92c2465922310916ad0c46ec9999295 + +LABEL maintainer="Antrea " +LABEL description="A Docker image with useful tools (for networking) based on Ubuntu" + +# Keep it in lexicographic order +ENV TOOLS apache2-utils \ + arping \ + conntrack \ + curl \ + dnsutils \ + ethtool \ + iperf3 \ + iproute2 \ + iptables \ + iputils-ping \ + ipset\ + kmod \ + ncat \ + netperf \ + nmap \ + socat \ + tcpdump \ + telnet \ + traceroute \ + wget + +# See https://github.com/kubernetes-sigs/iptables-wrappers +ADD --chmod=700 https://raw.githubusercontent.com/kubernetes-sigs/iptables-wrappers/e139a115350974aac8a82ec4b815d2845f86997e/iptables-wrapper-installer.sh /iptables-wrapper-installer.sh + +# We run /iptables-wrapper-installer.sh with --no-sanity-check to avoid an issue +# when building the arm64 version of this docker image with qemu: +# Failed to initialize nft: Protocol not supported +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $TOOLS && \ + rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \ + /iptables-wrapper-installer.sh --no-sanity-check + +COPY VERSION /VERSION + +# Use K8s pause binary as the default command. +# This is convenient when using this Docker image to run a K8s Pod which is meant to be exec'ed +# into, as there is no need to specify an explicit command when deploying the Pod. +# "pause" is superior to a simple "sleep infinity", as it handled signals correctly. +# Note that pause is a static binary. +COPY --from=registry.k8s.io/pause:3.9 /pause /pause + +CMD ["/pause"] diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index e6f5f676130..937eba802a5 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -108,7 +108,16 @@ function add_option { } function docker_run_with_host_net { - docker run --rm --net=host --privileged antrea/toolbox:latest "$@" + local default_image="antrea/toolbox:latest" + if [ "$#" -eq 1]; then + local image=$default_image + local command="$@" + else + local image="$1" + shift + local command="$@" + fi + docker run --rm --net=host --privileged $image $command } function configure_networks { @@ -123,9 +132,9 @@ function configure_networks { # Inject allow all iptables to preempt docker bridge isolation rules if [[ ! -z $SUBNETS ]]; then set +e - docker_run_with_host_net iptables -C DOCKER-USER -j ACCEPT > /dev/null 2>&1 + docker_run_with_host_net "iptables -C DOCKER-USER -j ACCEPT > /dev/null 2>&1" if [[ $? -ne 0 ]]; then - docker_run_with_host_net iptables -I DOCKER-USER -j ACCEPT + docker_run_with_host_net "iptables -I DOCKER-USER -j ACCEPT" fi set -e fi @@ -257,7 +266,7 @@ function update_kind_ipam_routes { return fi echo "$node_data"| while read pod_cidr node_ip; do - docker_run_with_host_net ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true + docker_run_with_host_net "ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true" done } @@ -279,27 +288,47 @@ function configure_vlan_subnets { vlan_interface="br-${bridge_id:0:7}.$vlan_id" vlan_interfaces+=("$vlan_interface") - docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id - docker_run_with_host_net ip link set $vlan_interface up + docker_run_with_host_net "ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id" + docker_run_with_host_net "ip link set $vlan_interface up" IFS=',' read -r -a subnet_array <<< "$subnets" for subnet in "${subnet_array[@]}" ; do echo "Configuring extra IP $subnet to VLAN interface $vlan_interface" - docker_run_with_host_net ip addr add dev $vlan_interface $subnet + docker_run_with_host_net "ip addr add dev $vlan_interface $subnet" done - docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT - docker_run_with_host_net iptables -t filter -A FORWARD -i $vlan_interface -o $bridge_interface -j ACCEPT - docker_run_with_host_net iptables -t filter -A FORWARD -i $vlan_interface -o $vlan_interface -j ACCEPT + docker_run_with_host_net "iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT" + docker_run_with_host_net "iptables -t filter -A FORWARD -i $vlan_interface -o $bridge_interface -j ACCEPT" + docker_run_with_host_net "iptables -t filter -A FORWARD -i $vlan_interface -o $vlan_interface -j ACCEPT" done # Allow traffic between VLANs for ((i=0; i<${#vlan_interfaces[@]}; i++)); do for ((j=i+1; j<${#vlan_interfaces[@]}; j++)); do - docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[i]} -o ${vlan_interfaces[j]} -j ACCEPT - docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT + docker_run_with_host_net "iptables -t filter -A FORWARD -i ${vlan_interfaces[i]} -o ${vlan_interfaces[j]} -j ACCEPT" + docker_run_with_host_net "iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT" done done + + # Adding ipset rule to prevent SNAT based on flexibleIPAM e2e tests + if [[ $FLEXIBLE_IPAM == true ]]; then + docker buildx build -t antrea/toolbox:ipset -f ./ci/kind/Dockerfile.ipset . + IMAGE="antrea/toolbox:ipset" + + docker_run_with_host_net "$IMAGE" "ipset create excluded_subnets hash:net" + docker_run_with_host_net "$IMAGE" "ipset add excluded_subnets 192.168.241.0/24" + docker_run_with_host_net "$IMAGE" "ipset add excluded_subnets 192.168.242.0/24" + docker_run_with_host_net "$IMAGE" "ipset list excluded_subnets" + docker_run_with_host_net "$IMAGE" "iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 192.168.240.0/24 -m set ! --match-set excluded_subnets dst -j MASQUERADE" + + docker_run_with_host_net "$IMAGE" "ipset create excluded_ipam_subnets hash:net" + docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.241.0/24" + docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.242.0/24" + docker_run_with_host_net "$IMAGE" "ipset add excluded_ipam_subnets 192.168.240.0/24" + docker_run_with_host_net "$IMAGE" "ipset list excluded_ipam_subnets" + docker_run_with_host_net "$IMAGE" "iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_ipam_subnets dst -j MASQUERADE" + fi + } function delete_vlan_subnets { @@ -309,13 +338,13 @@ function delete_vlan_subnets { bridge_interface="br-${bridge_id:0:12}" vlan_interface_prefix="br-${bridge_id:0:7}." - found_vlan_interfaces=$(docker_run_with_host_net ip -br link show type vlan | cut -d " " -f 1) + found_vlan_interfaces=$(docker_run_with_host_net "ip -br link show type vlan | cut -d " " -f 1") for interface in $found_vlan_interfaces ; do if [[ $interface =~ ${vlan_interface_prefix}[0-9]+@${bridge_interface} ]]; then interface_name=${interface%@*} - docker_run_with_host_net iptables -t filter -D FORWARD -i $bridge_interface -o $interface_name -j ACCEPT || true - docker_run_with_host_net iptables -t filter -D FORWARD -o $bridge_interface -i $interface_name -j ACCEPT || true - docker_run_with_host_net ip link del $interface_name + docker_run_with_host_net "iptables -t filter -D FORWARD -i $bridge_interface -o $interface_name -j ACCEPT || true" + docker_run_with_host_net "iptables -t filter -D FORWARD -o $bridge_interface -i $interface_name -j ACCEPT || true" + docker_run_with_host_net "ip link del $interface_name" fi done } @@ -327,6 +356,13 @@ function delete_networks { docker network rm $networks > /dev/null 2>&1 echo "deleted networks $networks" fi + + if [[ $FLEXIBLE_IPAM == true ]]; then + networks=$(docker network ls -f name=kind --format '{{.Name}}') + networks="$(echo $networks)" + docker network rm $networks > /dev/null 2>&1 + echo "deleted networks $networks" + fi } function load_images { @@ -711,7 +747,6 @@ if [[ $ACTION == "destroy" ]]; then exit fi - kind_version=$(kind version | awk '{print $2}') kind_version=${kind_version:1} # strip leading 'v' function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; } @@ -728,5 +763,10 @@ if [[ $ACTION == "create" ]]; then echoerr "Only one of '--subnets' and '--extra-networks' can be specified" exit 1 fi + + # To run FlexibleIPAM e2e tests, kind network should be in 192.168.240.0/24 subnet. + if [[ $FLEXIBLE_IPAM == true ]]; then + docker network create -d bridge --subnet 192.168.240.0/24 kind + fi create fi diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 1b9030c76df..4afece2077c 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -89,6 +89,7 @@ setup_only=false cleanup_only=false test_only=false run="" +flexible_ipam=false antrea_controller_image="antrea/antrea-controller-ubuntu" antrea_agent_image="antrea/antrea-agent-ubuntu" use_non_default_images=false @@ -110,6 +111,10 @@ case $key in proxy_all=true shift ;; + --flexible-ipam) + flexible_ipam=true + shift + ;; --no-kube-proxy) no_kube_proxy=true shift @@ -249,6 +254,10 @@ if $flow_visibility; then manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi +if $flexible_ipam; then + manifest_args="$manifest_args --flexible-ipam --multicast" +fi + COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \ "antrea/nginx:1.21.6-alpine" \ "antrea/toolbox:1.3-0") @@ -302,6 +311,10 @@ if $extra_vlan; then fi fi +if $flexible_ipam; then + vlan_args="$vlan_args --vlan-subnets 11=192.168.241.1/24 --vlan-subnets 12=192.168.242.1/24" +fi + function setup_cluster { args=$1 @@ -330,7 +343,11 @@ function setup_cluster { fi echo "creating test bed with args $args" - eval "timeout 600 $TESTBED_CMD create kind $args" + if $flexible_ipam; then + eval "timeout 600 $TESTBED_CMD --flexible-ipam create kind $args" + else + eval "timeout 600 $TESTBED_CMD create kind $args" + fi } function run_test { @@ -348,8 +365,13 @@ function run_test { timeout="80m" coverage_args="--coverage --coverage-dir $ANTREA_COV_DIR" else - $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml - $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml + if $flexible_ipam; then + $YML_CMD --flexible-ipam --multicast --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml + echo "debug-1" + else + $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml + $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml + fi timeout="75m" fi @@ -401,7 +423,15 @@ function run_test { EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips" fi - go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS + if $flexible_ipam; then + sudo iptables -t nat -vnL + kubectl get pods -o wide -A + ip route + export GO111MODULE=on + go test -v antrea.io/antrea/test/e2e --provider kind -timeout=100m --prometheus --antrea-ipam + else + go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS + fi if $coverage; then pushd $ANTREA_COV_DIR