e2e.yaml

name: E2E CI

on:
  release:
    types: [created]
  pull_request:
    branches:
      - main
      - v*
    paths:
      - "**.go"
      - "!**_test.go" # exclude test files to ignore unit test changes
      - "test/**" # include test files in e2e again
      - "!**.md"
      - "Dockerfile.release"
      - ".github/workflows/e2e.yaml"
      - "chart/**"
      - "manifests/**"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  REPOSITORY_NAME: ghcr.io/${{ github.repository }}-ci
  TAG_NAME: PR${{ github.event.number }}
  VCLUSTER_SUFFIX: vcluster
  VCLUSTER_NAME: vcluster
  VCLUSTER_NAMESPACE: vcluster

jobs:
  build-and-push-syncer-image:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - run: git fetch --force --tags
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
      - name: Setup Just
        uses: extractions/setup-just@v2
      - name: Setup Syft
        uses: anchore/sbom-action/download-syft@v0.17.8
      - name: Setup GoReleaser
        uses: goreleaser/goreleaser-action@v6
        with:
          install-only: true
          version: latest
      - name: Build and save syncer image
        run: |
          set -x
          TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster --clean --output ./vcluster
          docker build -t "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}" -f Dockerfile.release --build-arg TARGETARCH=amd64 --build-arg TARGETOS=linux .
          docker save -o vcluster_syncer "${{ env.REPOSITORY_NAME }}:${{ env.TAG_NAME }}"
      - name: Upload syncer image to artifact
        uses: actions/upload-artifact@v4
        with:
          name: vcluster_syncer
          path: ./vcluster_syncer
          retention-days: 7

  build-vcluster-cli:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - run: git fetch --force --tags
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
      - name: Setup Just
        uses: extractions/setup-just@v2
      - name: Setup Syft
        uses: anchore/sbom-action/download-syft@v0.17.8
      - name: Setup GoReleaser
        uses: goreleaser/goreleaser-action@v6
        with:
          install-only: true
      - name: Build vcluster cli
        run: |
          set -x
          TELEMETRY_PRIVATE_KEY="" goreleaser build --single-target --snapshot --id vcluster-cli --clean --output ./vcluster
      - name: Upload vcluster cli to artifact
        uses: actions/upload-artifact@v4
        with:
          name: vcluster
          path: ./vcluster
          retention-days: 7

  get-testsuites-dir:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - id: set-paths-matrix
        run: |
          set -x
          sudo apt-get install -y jq
          paths=$(ls -d ./test/e2e*)
          echo "matrix=$(printf '%s\n' "${paths}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
    outputs:
      matrix: ${{ steps.set-paths-matrix.outputs.matrix }}

  build-e2e:
    name: Build e2e binaries
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Configure git
        run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
        env:
          GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}

      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version-file: go.mod

      - name: Build e2e binary
        run: |
          go run -mod=vendor github.com/onsi/ginkgo/v2/ginkgo build --require-suite -r --mod vendor $(ls -d ./test/e2e* | jq -R . | jq -rcs '. | join(" \\\n")')
        env:
          GOWORK: off

      - name: Upload e2e binaries to artifacts
        uses: actions/upload-artifact@v4
        with:
          name: e2e-binaries
          path: ./test/*/*.test
          retention-days: 7

  vcluster-install-delete:
    name: Install and delete virtual cluster
    needs:
      - build-and-push-syncer-image
      - build-vcluster-cli

    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - uses: azure/setup-helm@v4
        name: Setup Helm
        with:
          version: "v3.11.0"

      - name: Set up kind k8s cluster
        uses: engineerd/setup-kind@v0.5.0
        with:
          version: "v0.20.0"
          image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e

      - name: Testing kind cluster set-up
        run: |
          set -x
          kubectl cluster-info
          kubectl get pods -n kube-system
          echo "kubectl config current-context:" $(kubectl config current-context)
          echo "KUBECONFIG env var:" ${KUBECONFIG}

      - name: Download vcluster cli
        uses: actions/download-artifact@v4
        with:
          name: vcluster

      - name: Download syncer image
        uses: actions/download-artifact@v4
        with:
          name: vcluster_syncer

      - name: Setup environment
        run: |

          kind load image-archive vcluster_syncer

          chmod +x vcluster && sudo mv vcluster /usr/bin

          sudo apt-get install -y sed

      - name: Run tests - install and delete virtual cluster using kubectl
        run: |
          set -x

          ./hack/vcluster-install-scripts/test-kubectl-install.sh

      - name: Run tests - install and delete virtual cluster using helm
        run: |
          set -x

          ./hack/vcluster-install-scripts/test-helm-install.sh

  download-latest-cli:
    name: Execute test suites
    runs-on: ubuntu-latest
    steps:
      - name: download current cli
        run: |
          curl -L -o vcluster-current "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64"
      - name: Upload vcluster cli to artifact
        uses: actions/upload-artifact@v4
        with:
          name: vcluster-current
          path: ./vcluster-current
          retention-days: 7

  upgrade-test:
    name: test if we can upgrade from older version
    needs:
      - build-and-push-syncer-image
      - build-vcluster-cli
      - download-latest-cli
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        distribution: ["k3s", "k8s", "k0s"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - uses: azure/setup-helm@v4
        name: Setup Helm
        with:
          version: "v3.11.0"
      - name: Set up kind k8s cluster
        uses: engineerd/setup-kind@v0.5.0
        with:
          version: "v0.20.0"
          image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e

      - name: Testing kind cluster set-up
        run: |
          set -x
          kubectl cluster-info
          kubectl get pods -n kube-system
          echo "kubectl config current-context:" $(kubectl config current-context)
          echo "KUBECONFIG env var:" ${KUBECONFIG}

      - name: Download vcluster cli
        uses: actions/download-artifact@v4
        with:
          name: vcluster
          path: vcluster-dev

      - name: Download current cli
        uses: actions/download-artifact@v4
        with:
          name: vcluster-current

      - name: Download syncer image
        uses: actions/download-artifact@v4
        with:
          name: vcluster_syncer

      - name: install sed
        run: |
          sudo apt-get install -y sed

      - name: Install yq@v4
        run: go install github.com/mikefarah/yq/v4@latest

      - name: create vcluster with current cli
        run: |
          chmod +x ./vcluster-current

          kind load image-archive vcluster_syncer
          yq eval '.controlPlane.distro.${{ matrix.distribution }}.enabled = true' > ./test/vcluster-current.yaml

          ./vcluster-current create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \
          --create-namespace \
          --debug \
          --connect=false \
          -f ./test/vcluster-current.yaml

          ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }}

      - name: upgrade with the dev cli
        run: |
          chmod +x ./vcluster-dev/vcluster
          set -x

          sed -i "s|REPLACE_REPOSITORY_NAME|${{ env.REPOSITORY_NAME }}|g" test/commonValues.yaml
          sed -i "s|REPLACE_TAG_NAME|${{ env.TAG_NAME }}|g" test/commonValues.yaml
          yq eval -i '.controlPlane.distro.${{ matrix.distribution }}.enabled = true' test/commonValues.yaml

          ./vcluster-dev/vcluster create vcluster \
          --connect=false \
          --upgrade \
          --local-chart-dir ./chart \
          -f ./test/commonValues.yaml

          ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }}

  e2e-tests:
    name: Execute test suites
    needs:
      - build-and-push-syncer-image
      - build-vcluster-cli
      - get-testsuites-dir
      - build-e2e

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        distribution: ["k3s", "k8s", "k0s"]
        test-suite-path: ${{fromJson(needs.get-testsuites-dir.outputs.matrix)}}
        multinamespace-mode: ["false", "true"]
        ha: ["false", "true"]
        include:
          - distribution: "k8s"
            ha: "true"
            test-suite-path: "./test/e2e"
            multinamespace-mode: "false"
          - distribution: "k3s"
            ha: "true"
            test-suite-path: "./test/e2e"
            multinamespace-mode: "false"
          - distribution: "k0s"
            ha: "true"
            test-suite-path: "./test/e2e"
            multinamespace-mode: "false"
        exclude:
          - ha: "true"
          - distribution: "k8s"
            multinamespace-mode: "true"
          - distribution: "k0s"
            multinamespace-mode: "true"
          - distribution: "k3s"
            multinamespace-mode: "true"
            test-suite-path: "./test/e2e_target_namespace"
          - distribution: "k3s"
            multinamespace-mode: "true"
            test-suite-path: "./test/e2e_plugin"
          - distribution: "k3s"
            multinamespace-mode: "true"
            test-suite-path: "./test/e2e_isolation_mode"

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - uses: azure/setup-helm@v4
        name: Setup Helm
        with:
          version: "v3.11.0"

      - name: Set up kind k8s cluster
        uses: engineerd/setup-kind@v0.5.0
        with:
          version: "v0.20.0"
          image: kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e

      - name: Testing kind cluster set-up
        run: |
          set -x
          kubectl cluster-info
          kubectl get pods -n kube-system
          echo "kubectl config current-context:" $(kubectl config current-context)
          echo "KUBECONFIG env var:" ${KUBECONFIG}

      - name: Download vcluster cli
        uses: actions/download-artifact@v4
        with:
          name: vcluster

      - name: Download syncer image
        uses: actions/download-artifact@v4
        with:
          name: vcluster_syncer

      - name: Download e2e binaries
        uses: actions/download-artifact@v4
        with:
          name: e2e-binaries
          path: ./test

      # - name: Setup upterm session for debugging
      #   uses: lhotari/action-upterm@v1

      - name: Create vcluster
        id: create-vcluster
        run: |
          set -x

          extraArgs=()
          if [ ${{ matrix.multinamespace-mode }} == "true" ]; then
            extraArgs+=( -f ./test/multins_values.yaml )
          fi
          if [ ${{ matrix.test-suite-path }} == "./test/e2e_metrics_proxy" ]; then
            helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
            helm upgrade --install metrics-server metrics-server/metrics-server --set args={--kubelet-insecure-tls} --set containerPort=4443 -n kube-system
          fi

          if [ ${{ matrix.ha }} == "true" ]; then
            haValues="-f ./test/values_ha.yaml"
          fi

          if [ ${{ matrix.test-suite-path }} == "./test/e2e_target_namespace" ]; then
            kubectl apply -f ${{ matrix.test-suite-path }}/role.yaml
          fi

          sudo apt-get install -y sed

          sed -i "s|REPLACE_REPOSITORY_NAME|${{ env.REPOSITORY_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml
          sed -i "s|REPLACE_TAG_NAME|${{ env.TAG_NAME }}|g" ${{ matrix.test-suite-path }}/../commonValues.yaml
          yq eval -i '.controlPlane.distro.${{ matrix.distribution }}.enabled = true'  ${{ matrix.test-suite-path }}/../commonValues.yaml

          kind load image-archive vcluster_syncer

          chmod +x vcluster && sudo mv vcluster /usr/bin

          vcluster create ${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} \
          --create-namespace \
          --debug \
          --connect=false \
          --local-chart-dir ./chart \
          -f ./test/commonValues.yaml \
          $haValues \
          -f ${{ matrix.test-suite-path }}/values.yaml \
          "${extraArgs[@]}"
        continue-on-error: true

      - name: Wait until vcluster is ready
        id: wait-until-vcluster-is-ready
        if: steps.create-vcluster.outcome == 'success'
        run: |
          set -x

          ./hack/wait-for-pod.sh -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }}

        continue-on-error: true

      - name: Collect deployment information in case vcluster fails to start
        if: steps.wait-until-vcluster-is-ready.outcome != 'success'
        run: |
          set -x
          kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }}
          echo "======================================================================================================================"
          kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp'
          echo "======================================================================================================================"
          kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1
          echo "======================================================================================================================"
          if [[ "${{ matrix.test-suite-path }}" = "./test/e2e_plugin" ]]; then
            kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c bootstrap-with-deployment --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c bootstrap-with-deployment --tail=-1
            echo "======================================================================================================================"
          fi
          kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }}
          exit 1

      # Skips NetworkPolicy tests because they require network plugin with support (e.g. Calico)
      - name: Execute e2e tests
        id: execute-e2e-tests
        working-directory: ${{ matrix.test-suite-path }}
        run: |
          set -x

          sudo chmod +x $(echo "${{ matrix.test-suite-path }}" | sed "s#./test/##g").test

          VCLUSTER_SUFFIX=${{ env.VCLUSTER_SUFFIX }} VCLUSTER_NAME=${{ env.VCLUSTER_NAME }} VCLUSTER_NAMESPACE=${{ env.VCLUSTER_NAMESPACE }} MULTINAMESPACE_MODE=${{ matrix.multinamespace-mode }} ./$(echo "${{ matrix.test-suite-path }}" | sed "s#./test/##g").test -test.v --ginkgo.v --ginkgo.skip='.*NetworkPolicy.*' --ginkgo.fail-fast
          if kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p >/dev/null 2>/dev/null; then
            echo "vCluster has restarted during testing, failing..."
            exit 1
          fi

        continue-on-error: true

      - name: Print logs if e2e tests fail
        if: steps.execute-e2e-tests.outcome == 'failure'
        run: |
          set -x
          kubectl get pods -o yaml -n ${{ env.VCLUSTER_NAMESPACE }}
          echo "======================================================================================================================"
          kubectl get events -n ${{ env.VCLUSTER_NAMESPACE }} --sort-by='.lastTimestamp'
          echo "======================================================================================================================"
          kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1 -p || kubectl logs -l app=${{ env.VCLUSTER_SUFFIX }} -n ${{ env.VCLUSTER_NAMESPACE }} -c syncer --tail=-1
          echo "======================================================================================================================"
          kubectl describe pods -n ${{ env.VCLUSTER_NAMESPACE }}
          exit 1