diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 168bf029b33..57a7d2d7655 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -277,6 +277,39 @@ jobs: path: log.tar.gz retention-days: 30 + test-e2e-ipam-feature-enabled: + name: E2e tests on a Kind cluster on Linux with FlexibleIPAM feature enabled + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest-4-cores] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: Download Antrea image from previous job + uses: actions/download-artifact@v4 + with: + name: antrea-ubuntu-cov + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu.tar + - name: Install Kind + run: | + KIND_VERSION=$(head -n1 ./ci/kind/version) + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run ipam e2e tests + run: | + ./ci/kind/test-e2e-kind.sh --flexible-ipam --encap-mode noEncap + test-e2e-noencap: name: E2e tests on a Kind cluster on Linux (noEncap) needs: [build-antrea-coverage-image] diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index e6f5f676130..622d3d48ff8 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -108,7 +108,9 @@ function add_option { } function docker_run_with_host_net { - docker run --rm --net=host --privileged antrea/toolbox:latest "$@" + local image=${1:-antrea/toolbox:latest} + shift + docker run --rm --net=host --privileged $image "$@" } function configure_networks { @@ -327,6 +329,13 @@ function delete_networks { docker network rm $networks > /dev/null 2>&1 echo "deleted networks $networks" fi + + if [[ $FLEXIBLE_IPAM == true ]]; then + networks=$(docker network ls -f name=kind --format '{{.Name}}') + networks="$(echo $networks)" + docker network rm $networks > /dev/null 2>&1 + echo "deleted networks $networks" + fi } function load_images { @@ -711,7 +720,6 @@ if [[ $ACTION == "destroy" ]]; then exit fi - kind_version=$(kind version | awk '{print $2}') kind_version=${kind_version:1} # strip leading 'v' function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; } @@ -728,5 +736,33 @@ if [[ $ACTION == "create" ]]; then echoerr "Only one of '--subnets' and '--extra-networks' can be specified" exit 1 fi + if [[ $FLEXIBLE_IPAM == true ]]; then + docker network create -d bridge --subnet 192.168.240.0/24 kind + IMAGE="janeczku/redwall-alpine-ipset:latest" + + # docker_run_with_host_net bash -c "echo 'deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) universe' >> /etc/apt/sources.list && apt-get update" + # # docker_run_with_host_net apt-get update + # docker_run_with_host_net apt-get install -y ipset iproute2 + # docker_run_with_host_net apt-get install -y ipset + # docker_run_with_host_net echo $PATH + # # docker_run_with_host_net modprobe ip_set + # # docker_run_with_host_net modprobe xt_set + + docker_run_with_host_net $IMAGE ipset create excluded_subnets hash:net + docker_run_with_host_net $IMAGE ipset add excluded_subnets 192.168.241.0/24 + docker_run_with_host_net $IMAGE ipset add excluded_subnets 192.168.242.0/24 + docker_run_with_host_net $IMAGE ipset list excluded_subnets + + bridge_id=$(docker network inspect kind -f {{.ID}}) + bridge_interface="br-${bridge_id:0:12}" + docker_run_with_host_net iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 192.168.240.0/24 -m set ! --match-set excluded_subnets dst -j MASQUERADE + + docker_run_with_host_net $IMAGE ipset create excluded_ipam_subnets hash:net + docker_run_with_host_net $IMAGE ipset add excluded_ipam_subnets 192.168.241.0/24 + docker_run_with_host_net $IMAGE ipset add excluded_ipam_subnets 192.168.242.0/24 + docker_run_with_host_net $IMAGE ipset add excluded_ipam_subnets 192.168.240.0/24 + docker_run_with_host_net $IMAGE ipset list excluded_ipam_subnets + docker_run_with_host_net iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_ipam_subnets dst -j MASQUERADE + fi create fi diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 1b9030c76df..4afece2077c 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -89,6 +89,7 @@ setup_only=false cleanup_only=false test_only=false run="" +flexible_ipam=false antrea_controller_image="antrea/antrea-controller-ubuntu" antrea_agent_image="antrea/antrea-agent-ubuntu" use_non_default_images=false @@ -110,6 +111,10 @@ case $key in proxy_all=true shift ;; + --flexible-ipam) + flexible_ipam=true + shift + ;; --no-kube-proxy) no_kube_proxy=true shift @@ -249,6 +254,10 @@ if $flow_visibility; then manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi +if $flexible_ipam; then + manifest_args="$manifest_args --flexible-ipam --multicast" +fi + COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \ "antrea/nginx:1.21.6-alpine" \ "antrea/toolbox:1.3-0") @@ -302,6 +311,10 @@ if $extra_vlan; then fi fi +if $flexible_ipam; then + vlan_args="$vlan_args --vlan-subnets 11=192.168.241.1/24 --vlan-subnets 12=192.168.242.1/24" +fi + function setup_cluster { args=$1 @@ -330,7 +343,11 @@ function setup_cluster { fi echo "creating test bed with args $args" - eval "timeout 600 $TESTBED_CMD create kind $args" + if $flexible_ipam; then + eval "timeout 600 $TESTBED_CMD --flexible-ipam create kind $args" + else + eval "timeout 600 $TESTBED_CMD create kind $args" + fi } function run_test { @@ -348,8 +365,13 @@ function run_test { timeout="80m" coverage_args="--coverage --coverage-dir $ANTREA_COV_DIR" else - $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml - $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml + if $flexible_ipam; then + $YML_CMD --flexible-ipam --multicast --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml + echo "debug-1" + else + $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml + $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec.yml + fi timeout="75m" fi @@ -401,7 +423,15 @@ function run_test { EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips" fi - go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS + if $flexible_ipam; then + sudo iptables -t nat -vnL + kubectl get pods -o wide -A + ip route + export GO111MODULE=on + go test -v antrea.io/antrea/test/e2e --provider kind -timeout=100m --prometheus --antrea-ipam + else + go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS + fi if $coverage; then pushd $ANTREA_COV_DIR