[TT-11578] Use bitnami charts for redis and mongo (#262)

* Use bitnami charts for redis and mongo * Fix command to install of Tyk OSS * Fix command * Use bitnami redis and mongodb to test Tyk Stack * Remove mongo version * Use bitnami redis for Tyk Data Plane chart * Try bitnami mongo version 13.18.5 * Print logs of dashboard and bootstrap job in case of failure * Ignore logs of test pod * Fix mongo url * Update mongo chart version * Differentiate between deploy and test failures * Induce failure * Fix condition * Add back ct install command for Tyk OSS * Induce error * Move out integration tests in different workflow * Add step for kind cluster creation * Rename workflow files * Use bitnami redis for smoke tests too * Upgrade github actions * Run integration tests against k8s v1.21 * Fix command * Add back integration tests to same workflow * Rename workflow * Use helm install and helm test for OSS --------- Co-authored-by: Burak Sekili <[email protected]>
TykTechnologies · Apr 29, 2024 · c6c8f1b · c6c8f1b
1 parent 11c007c
commit c6c8f1b
Showing 1 changed file with 125 additions and 96 deletions.
diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml
@@ -1,4 +1,4 @@
-name: Tests
+name: Run Tests
 
 on:
   pull_request:
@@ -19,12 +19,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.7
 
@@ -36,172 +36,201 @@ jobs:
       - name: Run chart-testing (lint)
         run: ct lint --config ct.yaml --all
 
+  smoke-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s_version: ["v1.26.13", "v1.27.10","v1.28.6","v1.29.1"]
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+
+      - name: Create Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: "kindest/node:${{ matrix.k8s_version }}"
+
+      - name: Install helm
+        uses: azure/setup-helm@v4
+
+      - name: Deploy Tyk OSS and Dependencies
+        run: |
+          kubectl create namespace ${{ env.TYK_OSS_NS }}
+
+          # Do not change the name
+          helm install redis redis --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }}
+
+          helm dependency update ./tyk-oss
+          helm install tyk-oss ./tyk-oss -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }} \
+            --set global.redis.addrs={redis-master.${{ env.TYK_OSS_NS }}.svc:6379} \
+            --set global.redis.passSecret.name=redis \
+            --set global.redis.passSecret.keyName=redis-password
+
+      - name: Upgrade Tyk OSS
+        run: |
+          helm upgrade -n ${{ env.TYK_OSS_NS }} tyk-oss ./tyk-oss --set tyk-gateway.gateway.kind=DaemonSet
+
+      - name: Uninstall Tyk OSS
+        run: |
+          helm uninstall tyk-oss -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }}
+          kubectl delete namespace ${{ env.TYK_OSS_NS }}
+
+      - name: Deploy Tyk Data Plane
+        run: |
+          kubectl create namespace ${{ env. TYK_DATAPLANE_NS }}
+
+          # Do not change the name
+          helm install redis redis --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
+
+          helm dependency update ./tyk-data-plane
+
+          helm install --namespace ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane ./tyk-data-plane --wait \
+            --set global.remoteControlPlane.orgId=${{ secrets.HYBRID_RPC_KEY }} \
+            --set global.remoteControlPlane.userApiKey=${{ secrets.HYBRID_API_KEY }} \
+            --set global.remoteControlPlane.connectionString=${{ secrets.HYBRID_MDCB_HOST }} \
+            --set global.redis.addrs={redis-master.${{ env.TYK_DATAPLANE_NS }}.svc:6379} \
+            --set global.redis.passSecret.name=redis \
+            --set global.redis.passSecret.keyName=redis-password
+
+      - name: Upgrade Tyk Data Plane
+        run: |
+          helm upgrade -n ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane ./tyk-data-plane --set tyk-gateway.gateway.kind=DaemonSet
+
+      - name: Uninstall Tyk Data Plane
+        run: |
+          helm uninstall tyk-data-plane -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
+          kubectl delete namespace ${{ env. TYK_DATAPLANE_NS }}
+          
   integration-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.11.2
+        uses: azure/setup-helm@v4
 
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-          check-latest: true
-
-      - name: Set up chart-testing
-        uses: helm/[email protected]
       - name: Create kind cluster
-        uses: helm/[email protected]
-
-      # Tyk OSS tests
-      - name: Deploy Tyk OSS and Dependencies
+        uses: helm/[email protected]
+        with:
+          node_image: "kindest/node:v1.29.1"
+
+      - name: Deploy OSS and Dependencies
+        id: deploy-oss
         run: |
           kubectl create namespace ${{ env.TYK_OSS_NS }}
           
           # Do not change the name
-          helm install redis simple-redis --wait --repo 'https://helm.tyk.io/public/helm/charts/' -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }}
+          helm install redis redis --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }} --version 19.0.2
           
           helm dependency update ./tyk-oss
 
-      - name: Run chart-testing (install)
-        run: ct install --namespace ${{ env.TYK_OSS_NS }} --charts ./tyk-oss --helm-extra-set-args "--wait=true"
+          helm install --namespace ${{ env.TYK_OSS_NS }} tyk-oss ./tyk-oss \
+          --set "global.redis.addrs={redis-master.${{ env.TYK_OSS_NS }}.svc:6379}" \
+          --set "global.redis.passSecret.name=redis" \
+          --set "global.redis.passSecret.keyName=redis-password" \
+          --timeout=${{ env.TIMEOUT }} 
 
+      - name: Run integration tests for Tyk OSS
+        id: test-oss
+        run: helm test -n ${{ env.TYK_OSS_NS }} tyk-oss
+
       - if: failure()
         name: Check logs in case of failure
-        run: kubectl logs -n ${{ env.TYK_OSS_NS}} tyk-oss-test-tyk-oss
+        run: |
+          if [ ${{ steps.deploy-oss.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env.TYK_OSS_NS }} gateway-tyk-oss-tyk-gateway
+          elif [ ${{ steps.test-oss.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env.TYK_OSS_NS }} tyk-oss-test-tyk-oss
+          fi
 
       - name: Uninstall Tyk OSS Dependencies
         run: |
           helm uninstall redis -n ${{ env.TYK_OSS_NS }} --timeout ${{ env.TIMEOUT }}
           kubectl delete namespace ${{ env.TYK_OSS_NS }}
-
-      # Tyk Stack tests
+      
       - name: Deploy Tyk Stack and dependencies
+        id: deploy-stack
         run: |
           kubectl create namespace ${{ env.TYK_STACK_NS }}
 
           # Do not change the name
           # Install redis
-          helm install redis simple-redis --wait --repo 'https://helm.tyk.io/public/helm/charts/' -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }}
+          helm install redis redis --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }} --version 19.0.2
          
           # Install mongo
-          helm install mongo simple-mongodb --wait --repo 'https://helm.tyk.io/public/helm/charts/' -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }}
+          helm install mongo mongodb --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }} --version 15.1.2
+
+          MONGODB_ROOT_PASSWORD=$(kubectl get secret -n ${{ env.TYK_STACK_NS }} mongo-mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 -d)
 
           helm dependency update ./tyk-stack
 
           helm install --namespace ${{ env.TYK_STACK_NS }} tyk-stack ./tyk-stack \
-            --set "global.mongo.mongoURL=mongodb://mongo.tyk-stack.svc:27017/tyk_analytics" \
-            --set "global.redis.addrs[0]=redis.tyk-stack.svc:6379" \
+            --set "global.mongo.mongoURL=mongodb://root:$MONGODB_ROOT_PASSWORD@mongo-mongodb.${{ env.TYK_STACK_NS }}.svc:27017/tyk_analytics?authSource=admin" \
+            --set "global.redis.addrs={redis-master.${{ env.TYK_STACK_NS }}.svc:6379"} \
+            --set "global.redis.passSecret.name=redis" \
+            --set "global.redis.passSecret.keyName=redis-password" \
             --set "global.license.dashboard=${{ secrets.DASH_LICENSE }}" \
             --set "global.storageType=mongo" \
             --timeout=${{ env.TIMEOUT }}
 
       - name: Run integration tests for Tyk Stack
+        id: test-stack
         run: helm test -n ${{ env.TYK_STACK_NS }} tyk-stack
 
       - if: failure()
         name: Check logs in case of failure
-        run: kubectl logs -n ${{ env.TYK_STACK_NS }} tyk-stack-test-tyk-stack
-
+        run: |
+          if [ ${{ steps.deploy-stack.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env.TYK_STACK_NS }} deployment/dashboard-tyk-stack-tyk-dashboard
+            kubectl logs -n ${{ env.TYK_STACK_NS }} job.batch/bootstrap-post-install-tyk-stack-tyk-bootstrap
+          elif [ ${{ steps.test-stack.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env.TYK_STACK_NS }} tyk-stack-test-tyk-stack
+          fi
+        
       - name: Uninstall Tyk Stack Dependencies
         run: |
           helm uninstall redis -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }}
           helm uninstall mongo -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }} 
           helm uninstall tyk-stack -n ${{ env.TYK_STACK_NS }} --timeout ${{ env.TIMEOUT }} 
           kubectl delete namespace ${{ env.TYK_STACK_NS }}
 
-      # Tyk Data Plane tests
       - name: Deploy Tyk Data Plane and Dependencies
+        id: deploy-data-plane
         run: |
           kubectl create namespace ${{ env. TYK_DATAPLANE_NS }}
           
           # Do not change the name
-          helm install redis simple-redis --wait --repo 'https://helm.tyk.io/public/helm/charts/' -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
+          helm install redis redis --wait --repo 'https://charts.bitnami.com/bitnami' -n ${{ env.TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }} --version 19.0.2
           
           helm dependency update ./tyk-data-plane
           
           helm install --namespace ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane ./tyk-data-plane --wait \
+            --set "global.redis.addrs={redis-master.${{ env. TYK_DATAPLANE_NS }}.svc:6379"} \
+            --set "global.redis.passSecret.name=redis" \
+            --set "global.redis.passSecret.keyName=redis-password" \
             --set global.remoteControlPlane.orgId=${{ secrets.HYBRID_RPC_KEY }} \
             --set global.remoteControlPlane.userApiKey=${{ secrets.HYBRID_API_KEY }} \
             --set global.remoteControlPlane.connectionString=${{ secrets.HYBRID_MDCB_HOST }}
 
       - name: Run integration tests for Tyk Data Plane
+        id: test-data-plane
         run: helm test -n ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane
 
       - if: failure()
         name: Check logs in case of failure
-        run: kubectl logs -n ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane-test-tyk-data-plane
+        run: |
+          if [ ${{ steps.deploy-data-plane.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env. TYK_DATAPLANE_NS }} deployment/gateway-tyk-data-plane-tyk-gateway
+          elif [ ${{ steps.test-data-plane.conclusion }} == 'failure' ]; then
+            kubectl logs -n ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane-test-tyk-data-plane
+          fi
 
       - name: Uninstall Tyk Data Plane Dependencies
         run: |
           helm uninstall tyk-data-plane -n ${{ env. TYK_DATAPLANE_NS }}
           helm uninstall redis -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
-          kubectl delete namespace ${{ env. TYK_DATAPLANE_NS }}
-
-  smoke-tests:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        k8s_version: ["v1.26.13", "v1.27.10","v1.28.6","v1.29.1"]
-    steps:
-      - name: Git Checkout
-        uses: actions/checkout@v3
-
-      - name: Create Kind Cluster
-        uses: helm/[email protected]
-        with:
-          node_image: "kindest/node:${{ matrix.k8s_version }}"
-
-      - name: Install helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.11.2
-
-      - name: Deploy Tyk OSS and Dependencies
-        run: |
-          kubectl create namespace tyk-oss
-
-          # Do not change the name
-          helm install redis simple-redis --repo 'https://helm.tyk.io/public/helm/charts/' -n tyk-oss --timeout ${{ env.TIMEOUT }}
-
-          helm dependency update ./tyk-oss
-          helm install tyk-oss ./tyk-oss -n tyk-oss --timeout ${{ env.TIMEOUT }}
-
-      - name: Upgrade Tyk OSS
-        run: |
-          helm upgrade -n tyk-oss tyk-oss ./tyk-oss --set tyk-gateway.gateway.kind=DaemonSet
-
-      - name: Uninstall Tyk OSS
-        run: |
-          helm uninstall tyk-oss -n tyk-oss --timeout ${{ env.TIMEOUT }}
-          kubectl delete namespace tyk-oss
-
-      - name: Deploy Tyk Data Plane
-        run: |
-          kubectl create namespace ${{ env. TYK_DATAPLANE_NS }}
-
-          # Do not change the name
-          helm install redis simple-redis --wait --repo 'https://helm.tyk.io/public/helm/charts/' -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
-
-          helm dependency update ./tyk-data-plane
-
-          helm install --namespace ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane ./tyk-data-plane --wait \
-            --set global.remoteControlPlane.orgId=${{ secrets.HYBRID_RPC_KEY }} \
-            --set global.remoteControlPlane.userApiKey=${{ secrets.HYBRID_API_KEY }} \
-            --set global.remoteControlPlane.connectionString=${{ secrets.HYBRID_MDCB_HOST }}
-
-      - name: Upgrade Tyk Data Plane
-        run: |
-          helm upgrade -n ${{ env. TYK_DATAPLANE_NS }} tyk-data-plane ./tyk-data-plane --set tyk-gateway.gateway.kind=DaemonSet
-
-      - name: Uninstall Tyk Data Plane
-        run: |
-          helm uninstall tyk-data-plane -n ${{ env. TYK_DATAPLANE_NS }} --timeout ${{ env.TIMEOUT }}
-          kubectl delete namespace ${{ env. TYK_DATAPLANE_NS }}
+          kubectl delete namespace ${{ env. TYK_DATAPLANE_NS }}