diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 1c40fd4d34..91b56273f6 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -1045,114 +1045,110 @@ jobs: cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - # validate-multi-namespace: - # runs-on: ubuntu-20.04 - # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] - # strategy: - # fail-fast: false - # matrix: - # cluster: [ - # {distribution: k3s, version: v1.24}, - # {distribution: k3s, version: v1.25}, - # {distribution: k3s, version: v1.26}, - # {distribution: kind, version: v1.27.0} - # ] - # env: - # APP_SLUG: multi-namespace-yeti - # steps: - # - name: Checkout - # uses: actions/checkout@v3 - - # - uses: azure/setup-helm@v3 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - - # - name: Create Cluster - # id: create-cluster - # uses: replicatedhq/replicated-actions/create-cluster@v1 - # with: - # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - # kubernetes-distribution: ${{ matrix.cluster.distribution }} - # kubernetes-version: ${{ matrix.cluster.version }} - # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - # timeout-minutes: '120' - # ttl: 2h - # export-kubeconfig: true + validate-multi-namespace: + runs-on: ubuntu-20.04 + needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] + strategy: + fail-fast: false + matrix: + cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + env: + APP_SLUG: multi-namespace-yeti + steps: + - name: Checkout + uses: actions/checkout@v3 - # - name: download kots binary - # uses: actions/download-artifact@v3 - # with: - # name: kots - # path: bin/ + - uses: azure/setup-helm@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} - # - run: chmod +x bin/kots + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + export-kubeconfig: true - # - name: run the test - # run: | - # set +e - # echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml - # ./bin/kots \ - # install "$APP_SLUG/automated" \ - # --license-file license.yaml \ - # --no-port-forward \ - # --namespace "$APP_SLUG" \ - # --shared-password password \ - # --kotsadm-registry ttl.sh \ - # --kotsadm-namespace automated-${{ github.run_id }} \ - # --kotsadm-tag 24h + - name: download kots binary + uses: actions/download-artifact@v3 + with: + name: kots + path: bin/ - # EXIT_CODE=$? - # if [ $EXIT_CODE -ne 0 ]; then - # echo "------pods:" - # kubectl -n "$APP_SLUG" get pods - # echo "------kotsadm logs" - # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - # exit $EXIT_CODE - # fi + - run: chmod +x bin/kots - # COUNTER=1 - # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - # ((COUNTER += 1)) - # if [ $COUNTER -gt 120 ]; then - # echo "Timed out waiting for app to be ready" - # ./bin/kots get apps --namespace "$APP_SLUG" - # echo "kotsadm logs:" - # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - # exit 1 - # fi - # sleep 1 - # done + - name: run the test + run: | + set +e + echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml + ./bin/kots \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --no-port-forward \ + --namespace "$APP_SLUG" \ + --shared-password password \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --kotsadm-tag 24h - # # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "------pods:" + kubectl -n "$APP_SLUG" get pods + echo "------kotsadm logs" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit $EXIT_CODE + fi - # if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then - # printf "postgresql helm release not found in postgres-test namespace\n\n" - # helm ls -n postgres-test - # exit 1 - # fi + COUNTER=1 + while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + ((COUNTER += 1)) + if [ $COUNTER -gt 120 ]; then + echo "Timed out waiting for app to be ready" + ./bin/kots get apps --namespace "$APP_SLUG" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit 1 + fi + sleep 1 + done + + # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly + + if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then + printf "postgresql helm release not found in postgres-test namespace\n\n" + helm ls -n postgres-test + exit 1 + fi - # if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then - # printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG" - # helm ls -n "$APP_SLUG" - # exit 1 - # fi + if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then + printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG" + helm ls -n "$APP_SLUG" + exit 1 + fi - # - name: Generate support bundle on failure - # if: failure() - # uses: ./.github/actions/generate-support-bundle - # with: - # kots-namespace: "$APP_SLUG" - # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle + with: + kots-namespace: "$APP_SLUG" + aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - # - name: Remove Cluster - # id: remove-cluster - # uses: replicatedhq/replicated-actions/remove-cluster@v1 - # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - # continue-on-error: true - # with: - # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} # validate-gke-install: @@ -2390,141 +2386,137 @@ jobs: # cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} - # validate-kots-upgrade: - # runs-on: ubuntu-20.04 - # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] - # strategy: - # fail-fast: false - # matrix: - # cluster: [ - # {distribution: k3s, version: v1.24}, - # {distribution: k3s, version: v1.25}, - # {distribution: k3s, version: v1.26}, - # {distribution: kind, version: v1.27.0} - # ] - # env: - # APP_SLUG: postgres-to-rqlite - # BASE_KOTS_VERSION: v1.57.0 - # steps: - # - name: Checkout - # uses: actions/checkout@v3 - - # - name: Create Cluster - # id: create-cluster - # uses: replicatedhq/replicated-actions/create-cluster@v1 - # with: - # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - # kubernetes-distribution: ${{ matrix.cluster.distribution }} - # kubernetes-version: ${{ matrix.cluster.version }} - # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - # timeout-minutes: '120' - # ttl: 2h - # export-kubeconfig: true - - # - name: download base kots version - # run: | - # curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ - # && tar zxvf kots_linux_amd64.tar.gz \ - # && mv kots "kots-$BASE_KOTS_VERSION" - - # - name: download kots binary - # uses: actions/download-artifact@v3 - # with: - # name: kots - # path: bin/ + validate-kots-upgrade: + runs-on: ubuntu-20.04 + needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ] + strategy: + fail-fast: false + matrix: + cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + env: + APP_SLUG: postgres-to-rqlite + BASE_KOTS_VERSION: v1.57.0 + steps: + - name: Checkout + uses: actions/checkout@v3 - # - run: chmod +x bin/kots + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + export-kubeconfig: true - # - name: run the test - # run: | - # set +e - # echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml + - name: download base kots version + run: | + curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ + && tar zxvf kots_linux_amd64.tar.gz \ + && mv kots "kots-$BASE_KOTS_VERSION" - # # install using the base KOTS version + - name: download kots binary + uses: actions/download-artifact@v3 + with: + name: kots + path: bin/ - # "./kots-$BASE_KOTS_VERSION" \ - # install "$APP_SLUG/automated" \ - # --license-file license.yaml \ - # --port-forward=false \ - # --namespace "$APP_SLUG" \ - # --shared-password password + - run: chmod +x bin/kots - # EXIT_CODE=$? - # if [ $EXIT_CODE -ne 0 ]; then - # echo "------pods:" - # kubectl -n "$APP_SLUG" get pods - # echo "------kotsadm logs" - # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - # exit $EXIT_CODE - # fi + - name: run the test + run: | + set +e + echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml - # COUNTER=1 - # while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - # ((COUNTER += 1)) - # if [ $COUNTER -gt 120 ]; then - # echo "Timed out waiting for app to be ready" - # "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" - # echo "kotsadm logs:" - # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - # exit 1 - # fi - # sleep 1 - # done + # install using the base KOTS version - # # upgrade using the new KOTS version + "./kots-$BASE_KOTS_VERSION" \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --port-forward=false \ + --namespace "$APP_SLUG" \ + --shared-password password + + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "------pods:" + kubectl -n "$APP_SLUG" get pods + echo "------kotsadm logs" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit $EXIT_CODE + fi - # ./bin/kots admin-console upgrade \ - # --namespace "$APP_SLUG" \ - # --kotsadm-registry ttl.sh \ - # --kotsadm-namespace automated-${{ github.run_id }} \ - # --kotsadm-tag 24h + COUNTER=1 + while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + ((COUNTER += 1)) + if [ $COUNTER -gt 120 ]; then + echo "Timed out waiting for app to be ready" + "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit 1 + fi + sleep 1 + done + + # upgrade using the new KOTS version + + ./bin/kots admin-console upgrade \ + --namespace "$APP_SLUG" \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --kotsadm-tag 24h - # # verify that the postgres to rqlite migration was successful + # verify that the postgres to rqlite migration was successful - # if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then - # echo "Failed to find a successful migration log line" - # echo "kotsadm logs:" - # kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" - # exit 1 - # fi + if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then + echo "Failed to find a successful migration log line" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" + exit 1 + fi - # # verify that the minio migration happened + # verify that the minio migration happened - # if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then - # echo "Failed to find initContainers in the kotsadm-minio statefulset" - # echo "kotsadm-minio statefulset:" - # kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml - # exit 1 - # fi + if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then + echo "Failed to find initContainers in the kotsadm-minio statefulset" + echo "kotsadm-minio statefulset:" + kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml + exit 1 + fi - # # make sure app is still installed and ready + # make sure app is still installed and ready - # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then - # echo "App is not ready after the upgrade" - # echo "kotsadm logs:" - # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - # exit 1 - # fi + if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then + echo "App is not ready after the upgrade" + echo "kotsadm logs:" + kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + exit 1 + fi - # printf "App is still installed and is ready after the migration\n\n" - # ./bin/kots get apps --namespace "$APP_SLUG" + printf "App is still installed and is ready after the migration\n\n" + ./bin/kots get apps --namespace "$APP_SLUG" - # - name: Generate support bundle on failure - # if: failure() - # uses: ./.github/actions/generate-support-bundle - # with: - # kots-namespace: "$APP_SLUG" - # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle + with: + kots-namespace: "$APP_SLUG" + aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - # - name: Remove Cluster - # id: remove-cluster - # uses: replicatedhq/replicated-actions/remove-cluster@v1 - # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - # continue-on-error: true - # with: - # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} # validate-kots-helm-release-secret-migration: