Migrate CI workflow to E2E self-hosted runners

genialis · Apr 29, 2024 · 2f2c5e3 · 2f2c5e3
1 parent 30900a7
commit 2f2c5e3
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 62 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -43,65 +43,7 @@ jobs:
           ${{ env.TOX_COMMAND }} -e ${{ matrix.toxenv }}
 
   e2e:
-    runs-on: arc-runner
-    needs: unittest
-    env:
-      GENESIS_POSTGRESQL_NAME: genialis-base
-      GENESIS_POSTGRESQL_USER: genialis-base
-      GENESIS_POSTGRESQL_PASSWORD: genialis-base
-      GENESIS_POSTGRESQL_PORT: 55434
-      GENESIS_REDIS_PORT: 56381
-      GENESIS_RESDK_PATH: ".."
-      GENESIS_REST_THROTTLING: '{"login": "100000/hour", "credentials": "100000/hour", "email": "100000/hour"}'
-
-    services:
-      postgres:
-        image: public.ecr.aws/docker/library/postgres:14
-        env:
-          POSTGRES_USER: genialis-base
-          POSTGRES_DB: genialis-base
-          POSTGRES_PASSWORD: genialis-base
-        ports:
-          - 55434:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-      redis:
-        image: public.ecr.aws/docker/library/redis:7-alpine
-        ports:
-          - 56381:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout resolwe-bio-py
-        uses: actions/checkout@v4
-
-      - name: Checkout genialis-base
-        uses: actions/checkout@v4
-        with:
-          repository: genialis/genialis-base
-          path: genialis-base
-          ssh-key: ${{ secrets.GHA_SSH_PRIVATE_KEY_GENIALIS_BASE }}
-
-      - name: Run tests
-        run: |
-          cd genialis-base
-
-          # Prepare frontend configuration
-          mkdir -p frontend/genjs/schema
-          echo '{}' > frontend/genjs/schema/configuration.json
-
-          # Make sure `python` refers to the tox environment's Python
-          export PATH=`pwd`/.tox/py310-e2e-resdk/bin:$PATH
-
-          tox --skip-missing-interpreters false -e py310-e2e-resdk
+    uses: ./.github/workflows/e2e.yaml
 
   build:
     runs-on: arc-runner

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -1,11 +1,111 @@
 name: ReSDK E2E
 
 on:
-  pull_request:
   workflow_dispatch:
+  workflow_call:
+
+env:
+  REGISTRY: ${{ vars.REGISTRY || '396487289173.dkr.ecr.eu-west-1.amazonaws.com' }}
+  AWS_REGION: ${{ vars.AWS_REGION || 'eu-west-1' }}
+  K3D_BASE_IMAGE: genialis/ci/k3d-base:v1.28.8-k3s1
+  GENESIS_POSTGRESQL_NAME: genialis-base
+  GENESIS_POSTGRESQL_USER: genialis-base
+  GENESIS_POSTGRESQL_PASSWORD: genialis-base
+  GENESIS_POSTGRESQL_PORT: 55434
+  GENESIS_REDIS_PORT: 56381
+  GENESIS_RESDK_PATH: ".."
+  GENESIS_REST_THROTTLING: '{"login": "100000/hour", "credentials": "100000/hour", "email": "100000/hour"}'
 
 jobs:
   e2e:
-    runs-on: e2e-runner
+    runs-on: ${{ vars.RUNNER_SCALE_SET }}
+
     steps:
-      - run: echo "Running."
+      - name: Checkout resolwe-bio-py
+        uses: actions/checkout@v4
+
+      - name: Reset Docker credentials store
+        run: |
+          # Docker is configured to use the ecr credentials helper,
+          # which clashes with the aws-actions/configure-aws-credentials action.
+          rm -f ~/.docker/config.json
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.REGISTRY }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Show versions
+        run: |
+          helmfile --version
+          helm version
+          kubectl version --client
+          sops --version
+          velero version
+
+      - name: Create K3d cluster
+        run: |
+          aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ vars.REGISTRY }}
+          # Fix authentication error by prefetching k3d base image
+          docker pull ${{ vars.REGISTRY }}/${{ env.K3D_BASE_IMAGE }}
+          # TODO: Add a way to override k3d-config.yaml base image tag
+          k3d cluster create --config e2e/k3d/k3d-config.yaml
+
+      - name: Add hosts to /etc/hosts
+        run: |
+          sudo echo "172.18.0.2 ws.local.genialis.io local.genialis.io" | sudo tee -a /etc/hosts
+
+      - name: Deploy Helm charts
+        run: |
+          cd /workdir
+          helmfile -e local -f helmfile-service.yaml sync
+
+      - name: Authorize cluster to pull private repositories
+        run: |
+          kubectl config set-context --current --namespace=default
+          kubectl get pods
+          kubectl wait pods -l app=pod-identity-webhook --for condition=Ready --timeout=1m
+          kubectl get cronjobs
+          kubectl create job --from=cronjob/ecr-auth -n default ecr-auth-manual
+          kubectl wait --for=condition=complete --timeout=1m job/ecr-auth-manual
+          kubectl rollout restart deployment -n velero velero
+
+      - name: Restore Velero edge snapshot
+        run: |
+          echo "List backups"
+          velero backup get
+          echo "Restore"
+          velero restore create --include-namespaces default --from-backup edge --wait
+          echo "Get restore"
+          velero restore get
+          RESTORE_NAME=$(velero restore get | grep edge | awk '{print $1}')
+          echo "Describe restore"
+          velero restore describe $RESTORE_NAME
+          echo "Logs restore"
+          velero restore logs $RESTORE_NAME
+
+      - name: Try the cluster
+        run: |
+          kubectl wait --for=condition=available --timeout=1200s -n default deployment/postgres
+          kubectl rollout restart deployment asgi-server channels-manager listener background-task uploader
+          kubectl wait --for=condition=available --timeout=1200s --all deployments
+          kubectl get pods -A
+          kubectl get pvc
+          kubectl get pv
+
+      - name: Test curl
+        run: |
+          ping local.genialis.io -c 1
+
+      - name: Run tests
+        run: |
+          # Make sure `python` refers to the tox environment's Python
+          export PATH=`pwd`/.tox/py310-e2e-resdk/bin:$PATH
+
+          tox --skip-missing-interpreters false -e py310-e2e-resdk