diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index feb8a94071..138383a61f 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -598,2798 +598,3285 @@ jobs: versions-to-test: ${{ steps.cmx-versions-to-test.outputs.versions-to-test }} - validate-existing-online-install-minimal: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: | - docker load -i e2e/bin/e2e-deps.tar - chmod +x e2e/bin/* - chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - kots-namespace: 'qakotsregression' - test-focus: 'Regression' - k8s-distribution: k3s - k8s-version: v1.26 - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-smoke-test: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'Smoke Test' - kots-namespace: 'smoke-test' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-minimal-rbac: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0}, - {distribution: openshift, version: 4.13.0-okd} - ] - env: - APP_SLUG: minimal-rbac - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h | tee output.txt - - if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then - echo "Expected to see an RBAC error for preflight checks, but did not" - exit 1 - fi - - if ! grep -q 'The app was not deployed.' output.txt; then - printf "Expected to see message about app not being deployed, but did not\n" - exit 1 - fi - - if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then - printf "Did not expect to see a failure about number of nodes, but did\n" - exit 1 - fi - - if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm role not found in namespace $APP_SLUG" - kubectl get role -n "$APP_SLUG" - exit 1 - fi - - if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm rolebinding not found in namespace $APP_SLUG" - kubectl get rolebinding -n "$APP_SLUG" - exit 1 - fi - - if kubectl get clusterrole | grep -q kotsadm; then - echo "found kotsadm clusterrole in minimal RBAC install" - kubectl get clusterrole - exit 1 - fi - - if kubectl get clusterrolebinding | grep -q kotsadm; then - echo "found kotsadm clusterrolebinding in minimal RBAC install" - kubectl get clusterrolebinding - exit 1 - fi - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-backup-and-restore: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'Backup and Restore' - kots-namespace: 'backup-and-restore' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-no-required-config: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'No Required Config' - kots-namespace: 'no-required-config' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-strict-preflight-checks: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: strict-preflight-checks - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --skip-preflights \ - --kotsadm-tag 24h | tee output.txt - - if ! grep -q 'The app was not deployed.' output.txt; then - printf "Expected to see message about app not being deployed, but did not\n" - exit 1 - fi - - if ! grep FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then - printf "Expected to see a failure about kubernetes version, but did not\n" - exit 1 - fi - - if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then - echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs" - echo "kotsadm logs:" - kubectl logs deploy/kotsadm -n "$APP_SLUG" - exit 1 - fi - - # disable the strict preflight check and app should deploy successfully - ./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - printf "App is installed successfully and is ready\n\n" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-config: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'Config Validation' - kots-namespace: 'config-validation' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-version-history-pagination: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: k3s-local, version: v1.27.1-k3s1} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'Version History Pagination' - kots-namespace: 'version-history-pagination' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-change-license: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'Change License' - kots-namespace: 'change-license' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-minimal-rbac-override: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0}, - {distribution: openshift, version: 4.13.0-okd} - ] - env: - APP_SLUG: minimal-rbac - APP_VERSION_LABEL: "0.0.1" - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: minimal rbac override on command line - run: | - set +e - echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --app-version-label "$APP_VERSION_LABEL" \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h \ - --use-minimal-rbac | tee output.txt - - if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then - echo "Expected to see an RBAC error for preflight checks, but did not" - exit 1 - fi - - if ! grep -q 'The app was not deployed.' output.txt; then - printf "Expected to see message about app not being deployed, but did not\n" - exit 1 - fi - - if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then - printf "Did not expect to see a failure about number of nodes, but did\n" - exit 1 - fi - - if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm role not found in namespace $APP_SLUG" - kubectl get role -n "$APP_SLUG" - exit 1 - fi - - if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm rolebinding not found in namespace $APP_SLUG" - kubectl get rolebinding -n "$APP_SLUG" - exit 1 - fi - - if kubectl get clusterrole | grep -q kotsadm; then - echo "found kotsadm clusterrole in minimal RBAC install" - kubectl get clusterrole - exit 1 - fi - - if kubectl get clusterrolebinding | grep -q kotsadm; then - echo "found kotsadm clusterrolebinding in minimal RBAC install" - kubectl get clusterrolebinding - exit 1 - fi - - - name: create namespace and dockerhub secret - run: | - kubectl delete ns "$APP_SLUG" --ignore-not-found - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: no minimal rbac override on command line - run: | - set +e - echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --app-version-label "$APP_VERSION_LABEL" \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h | tee output.txt - - if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then - echo "Did not expect to see an RBAC error for preflight checks, but did" - exit 1 - fi - - if ! grep -q 'The app was not deployed.' output.txt; then - printf "Expected to see message about app not being deployed, but did not\n" - exit 1 - fi - - if ! grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then - printf "Expected to see a failure about number of nodes, but did not\n" - exit 1 - fi - - if kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm role found in cluster scoped install" - kubectl get role -n "$APP_SLUG" - exit 1 - fi - - if kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then - echo "kotsadm rolebinding found in cluster scoped install" - kubectl get rolebinding -n "$APP_SLUG" - exit 1 - fi - - if ! kubectl get clusterrole | grep -q kotsadm; then - echo "No kotsadm clusterrole found in cluster scoped install" - kubectl get clusterrole - exit 1 - fi - - if ! kubectl get clusterrolebinding | grep -q kotsadm; then - echo "No kotsadm clusterrolebinding found in cluster scoped install" - kubectl get clusterrolebinding - exit 1 - fi - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-multi-namespace: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} - env: - APP_SLUG: multi-namespace-yeti - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: azure/setup-helm@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 180 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly - - if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then - printf "postgresql helm release not found in postgres-test namespace\n\n" - helm ls -n postgres-test - exit 1 - fi - - if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then - printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG" - helm ls -n "$APP_SLUG" - exit 1 - fi - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-kots-pull: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} - env: - APP_NAME: multi-namespace-yeti - APP_SLUG: multi-namespace - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_NAME" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_NAME" - - - name: run kots pull - run: | - set +e - echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml - ./bin/kots pull "$APP_NAME/automated" \ - --license-file license.yaml \ - --shared-password password \ - --namespace "$APP_NAME" \ - --exclude-admin-console - - kubectl create ns "$APP_NAME" - kubectl create ns nginx-test - kubectl create ns redis-test - kubectl create ns postgres-test - - # HACK: without operator, additonal namespaces don't get image pull secrets - echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} | base64 -d > replicated-registry-auth.json - kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json - kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json - kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json - kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json - kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json - - kustomize build "$PWD/$APP_SLUG/overlays/midstream" | kubectl apply -f - - kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" | kubectl apply -f - - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "Failed to apply spec" - echo "------pods:" - kubectl get pods -A - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - echo "Waiting for pods to start" - - COUNTER=1 - while [ "$(kubectl get pods --no-headers | grep -v Running | grep -cv Completed)" -gt 0 ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for pods to start" - kubectl get pods -A - exit 1 - fi - sleep 1 - done - - echo "All pods started" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_NAME" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-app-version-label: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: app-version-label - APP_VERSION_LABEL: v1.0.0 - LATEST_APP_VERSION_LABEL: v1.0.1 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: test kots install with version label - run: | - set +e - echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --app-version-label "$APP_VERSION_LABEL" \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL" - ./bin/kots get apps --namespace "$APP_SLUG" - - # test setting DockerHub credentials - - set +e - - # TODO: deploy and check secrets are actually created and images are pulled - ./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG" - ./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG" - if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then - echo "Found DockerHub secret in ${APP_SLUG} latest version" - else - echo "No DockerHub secret found in appication namespace" - exit 1 - fi - - - name: remove the app - run: | - set +e - ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force - - - name: test kots install without version label - run: | - set +e - echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-helm-install-order: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: helm-install-order - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --skip-preflights \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - printf "App is installed successfully and is ready\n\n" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-no-redeploy-on-restart: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: no-redeploy-on-restart - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --skip-preflights \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - # wait for application job to be created - COUNTER=1 - while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" == "0" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 60 ]; then - echo "Timed out waiting for job to be created" - exit 1 - fi - sleep 1 - done - - # delete the application job and restart the admin console - kubectl delete jobs -n "$APP_SLUG" --all - kubectl delete pods -l app=kotsadm -n "$APP_SLUG" - - # wait for old kotsadm pod to terminate - COUNTER=1 - while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" | awk 'NR>1' | wc -l)" != "1" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 60 ]; then - echo "More than 1 kotsadm pod found" - exit 1 - fi - sleep 1 - done - - # wait for new kotsadm pod to become ready - kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s - - # delay in case the app takes a bit to be deployed - sleep 20 - - # validate that the application wasn't re-deployed and the job wasn't re-created - if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" != "0" ]; then - echo "App should not be re-deployed after restart" - exit 1 - fi - - printf "Success. App was not re-deployed after restart\n\n" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-kubernetes-installer-preflight: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: kubernetes-installer-preflight - steps: - - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - - echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} | base64 -d > license.yaml - - # Fake kurl installation using the crd and installer spec from the application manifests: - - # Pull application manifests - ./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password - - # Apply installer crd - kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml" - - # Wait for crd to be created - kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh - - # Seems that the above does not always guarantee the crd exists? So just in case... - sleep 10 - - # Apply installer - kubectl apply -f "$APP_SLUG/upstream/installer.yaml" - - # Create kurl-config configmap in kube-system - kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094 - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - # validate that preflight checks ran - JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}" - if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then - echo "Preflight checks did not run" - exit 1 - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # try get apps without namespace (using kubeconfig) - # validate that output is the same as above - mkdir -p /tmp/.kube - sudo cp "$KUBECONFIG" /tmp/.kube/config - sudo chmod -R 777 /tmp/.kube - export KUBECONFIG=/tmp/.kube/config - kubectl config set-context --current --namespace="$APP_SLUG" - if [ "$(./bin/kots get apps | awk 'NR>1{print $2}')" != "ready" ]; then - echo "kots get apps output is not the same as above" - exit 1 - fi - - printf "App is installed successfully and is ready\n\n" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-kots-push-images-anonymous: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: run kots admin-console push-images - run: | - set +e - ./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }} - - - validate-kots-admin-console-generate-manifests: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: minimal-rbac - BASE_KOTS_VERSION: v1.72.0 - NAMESPACE: generate-manifests - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: run kots admin-console generate-manifests without k8s context - run: | - set +e - ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password - - - name: validate that ./admin-console exists and is not empty - run: | - set +e - if [ ! -d ./admin-console ]; then - echo "admin-console directory does not exist" - exit 1 - fi - if [ -z "$(ls -A ./admin-console)" ]; then - echo "admin-console directory is empty" - exit 1 - fi - - - name: remove admin-console directory - run: rm -rf ./admin-console - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download base kots version - run: | - curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ - && tar zxvf kots_linux_amd64.tar.gz \ - && mv kots "kots-$BASE_KOTS_VERSION" - - - name: create namespace - run: | - set +e - kubectl create namespace "$NAMESPACE" - - - name: run kots admin-console generate-manifests using base kots version with k8s context - run: | - set +e - "./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password - - - name: apply the generated manifests - run: | - set +e - kubectl apply -f ./admin-console -n "$NAMESPACE" - - - name: wait for the kotsadm-minio-0 pod to be created - run: | - set +e - COUNTER=1 - while ! kubectl get pods -n "$NAMESPACE" | grep -q kotsadm-minio-0; do - ((COUNTER += 1)) - if [ $COUNTER -gt 30 ]; then - echo "timed out waiting for kotsadm-minio-0 pod to be created" - exit 1 - fi - sleep 1 - done - - - name: wait for kotsadm-minio-0 pod to be ready - run: | - set +e - kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" - - - name: wait for the kotsadm deployment to be ready - run: | - set +e - kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" - - - name: run kots admin-console generate-manifests using new kots version with k8s context - run: | - set +e - ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - - name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed - run: | - set +e - if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then - echo "admin-console/minio-statefulset.yaml does not have initContainers" - exit 1 - fi - - - name: apply the generated manifests - run: | - set +e - kubectl apply -f ./admin-console -n "$NAMESPACE" - - - name: wait for kotsadm-minio-0 pod to be ready - run: | - set +e - sleep 10 - kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" - - - name: wait for the kotsadm deployment to be ready - run: | - set +e - kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" - - - name: run kots admin-console generate-manifests using new kots version with k8s context - run: | - set +e - ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - - name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed - run: | - set +e - if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then - echo "admin-console/minio-statefulset.yaml has initContainers" - exit 1 - fi - - - name: apply the generated manifests - run: | - set +e - kubectl apply -f ./admin-console -n "$NAMESPACE" - - - name: wait for kotsadm-minio-0 pod to be ready - run: | - set +e - sleep 10 - kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" - - - name: wait for the kotsadm deployment to be ready - run: | - set +e - kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" - - - name: print pods and logs on failure - if: failure() - run: | - echo "------pods:" - kubectl -n "$NAMESPACE" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE" - echo "------kotsadm-minio logs" - kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: automated-${{ github.run_id }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-min-kots-version: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] - env: - APP_SLUG: min-kots-version - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: | - docker load -i e2e/bin/e2e-deps.tar - chmod +x e2e/bin/* - chmod +x bin/* - cp ./bin/kots /usr/local/bin/kubectl-kots - sudo apt-get update -y && sudo apt-get install jq -y - - uses: ./.github/actions/kots-e2e - id: kots-e2e - with: - test-focus: 'Min KOTS Version' - kots-namespace: 'min-kots-version' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - kotsadm-image-registry: ttl.sh - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - k8s-cluster-skip-teardown: true - - - name: validate that kots install fails early - run: | - set +e - - result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null) - echo "$result" - - if [[ "$result" == *"requires"* ]] && [[ "$result" == *"10000.0.0"* ]]; then - exit 0 - else - exit 1 - fi - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} - - - validate-target-kots-version: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: | - docker load -i e2e/bin/e2e-deps.tar - chmod +x e2e/bin/* - chmod +x bin/* - cp ./bin/kots /usr/local/bin/kubectl-kots - sudo apt-get update -y && sudo apt-get install jq -y - - uses: ./.github/actions/kots-e2e - id: kots-e2e - with: - test-focus: 'Target KOTS Version' - kots-namespace: 'target-kots-version' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - k8s-cluster-skip-teardown: true - - - name: validate that kots install fails early - run: | - set +e - - result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null) - echo "$result" - - if [[ "$result" == *"requires"* ]] && [[ "$result" == *"1.0.0"* ]]; then - exit 0 - else - exit 1 - fi - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} - - - validate-range-kots-version: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: | - docker load -i e2e/bin/e2e-deps.tar - chmod +x e2e/bin/* - chmod +x bin/* - cp ./bin/kots /usr/local/bin/kubectl-kots - sudo apt-get update -y && sudo apt-get install jq -y - - uses: ./.github/actions/kots-e2e - id: kots-e2e - with: - test-focus: 'Range KOTS Version' - kots-namespace: 'range-kots-version' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - k8s-cluster-skip-teardown: true - - - name: validate that kots install fails early - run: | - set +e - - result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null) - echo "$result" - - if [[ "$result" == *"requires"* ]] && [[ "$result" == *"11000.0.0"* ]]; then - exit 0 - else - exit 1 - fi - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} - - - validate-kots-upgrade: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} - env: - APP_SLUG: postgres-to-rqlite - BASE_KOTS_VERSION: v1.57.0 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} - export-kubeconfig: true - - - name: download base kots version - run: | - curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ - && tar zxvf kots_linux_amd64.tar.gz \ - && mv kots "kots-$BASE_KOTS_VERSION" - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml - - # install using the base KOTS version - - "./kots-$BASE_KOTS_VERSION" \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --port-forward=false \ - --namespace "$APP_SLUG" \ - --shared-password password - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # upgrade using the new KOTS version - - ./bin/kots admin-console upgrade \ - --namespace "$APP_SLUG" \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - # verify that the postgres to rqlite migration was successful - - if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then - echo "Failed to find a successful migration log line" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" - exit 1 - fi - - # verify that the minio migration happened - - if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then - echo "Failed to find initContainers in the kotsadm-minio statefulset" - echo "kotsadm-minio statefulset:" - kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml - exit 1 - fi - - # make sure app is still installed and ready - - if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then - echo "App is not ready after the upgrade" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - printf "App is still installed and is ready after the migration\n\n" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-kots-helm-release-secret-migration: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: helm-release-secret-migration - RELEASE_NAME: helm-release-chart - RELEASE_NAMESPACE: helm-release - BASE_KOTS_VERSION: v1.94.0 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download base kots version - run: | - curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ - && tar zxvf kots_linux_amd64.tar.gz \ - && mv kots "kots-$BASE_KOTS_VERSION" - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} | base64 -d > license.yaml - - # install using the base KOTS version - "./kots-$BASE_KOTS_VERSION" \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward=true \ - --namespace "$APP_SLUG" \ - --shared-password password - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # verify that the helm release secret is created in the kotsadm namespace - releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}') - if [ -z "$releaseSecretName" ]; then - echo "Failed to find the helm release secret in the $APP_SLUG namespace" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - # if there are more than one helm release secrets, fail - if [ "$(echo "$releaseSecretName" | wc -l)" -gt 1 ]; then - echo "Found more than one helm release secret in the $APP_SLUG namespace" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - # upgrade using the new KOTS version - ./bin/kots admin-console upgrade \ - --namespace "$APP_SLUG" \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - # make a config change and redeploy the app - ./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG" - - # make sure app is still installed and ready - if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then - echo "App is not ready after the upgrade" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - COUNTER=1 - while [ "$(helm ls -n "$RELEASE_NAMESPACE" | grep "$RELEASE_NAME" | awk 'NR>0{print $3}')" != "2" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2" - helm ls -n "$RELEASE_NAMESPACE" - exit 1 - fi - sleep 1 - done - - # verify that the helm release secret is created in the helm relase namespace - COUNT=1 - releaseSecret="" - while [ -z "$releaseSecret" ]; do - ((COUNT += 1)) - if [ $COUNT -gt 10 ]; then - echo "Timed out waiting for the helm release secret to be migrated" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE") - done - - # verify that the release secret in app namepspace is deleted - oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG") - if [ -n "$oldReleaseSecret" ]; then - echo "Found the helm release secret in the $APP_SLUG namespace" - echo "$oldReleaseSecret" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - # verify that there are two helm release secrets in the helm release namespace - releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}') - releaseSecretNamesCount=$(echo "$releaseSecretNames" | wc -w) - if [ "$releaseSecretNamesCount" -ne 2 ]; then - echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)" - kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - - printf "Helm release secret migration test passed\n\n" - ./bin/kots get apps --namespace "$APP_SLUG" - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-multi-app-backup-and-restore: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'multi-app-backup-and-restore' - kots-namespace: 'multi-app-backup-and-restore' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-multi-app-install: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'multi-app-install' - kots-namespace: 'multi-app-install' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-airgap-smoke-test: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: download e2e deps - uses: actions/download-artifact@v3 - with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e - with: - test-focus: 'airgap-smoke-test' - kots-namespace: 'airgap-smoke-test' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} - kots-airgap: true - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' - - - validate-remove-app: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0, instance-type: r1.medium} - ] - env: - APP_SLUG: remove-app - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - instance-type: ${{ matrix.cluster.instance-type }} - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - - echo ${{ secrets.REMOVE_APP_LICENSE }} | base64 -d > license.yaml - HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG") - APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG") - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # test that --undeploy deletes application resources from the cluster - - ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy - - if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" - exit 1 - fi - - if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then - printf "Application resources are still found in the cluster\n\n" - kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" - exit 1 - fi - - if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then - printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n" - kubectl get crd rabbitmqclusters.rabbitmq.com - exit 1 - fi - - for ns in "${HELM_APP_NAMESPACES[@]}"; do - if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then - printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns" - kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 - exit 1 - fi - done - - # re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force - - if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" - exit 1 - fi - - for ns in "${APP_NAMESPACES[@]}"; do - if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then - printf "%s namespace does not contain any resources\n\n" "$ns" - kubectl get all,secrets,configmap,pvc -n "$ns" - exit 1 - fi - done - - if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then - printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n" - kubectl get crd rabbitmqclusters.rabbitmq.com - exit 1 - fi - - if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then - printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n" - kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system - exit 1 - fi - - for ns in "${HELM_APP_NAMESPACES[@]}"; do - if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then - printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns" - kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 - exit 1 - fi - done - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-registry-check: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: run the test - run: | - set +e - - ./bin/kots admin-console push-images \ - not-an-app.airgap localhost:1234/not-a-namespace \ - --registry-username not-a-username \ - --registry-password not-a-password > output.txt 2>&1 - - if ! grep -q 'Failed to test access' output.txt; then - printf "Expected registry validation to fail before pushing images, but did not.\n\n" - cat output.txt - exit 1 - fi - - rm output.txt - - ./bin/kots install not-an-app \ - --airgap-bundle not-an-app.airgap \ - --kotsadm-registry localhost:1234/not-a-namespace \ - --registry-username not-a-username \ - --registry-password not-a-password > output.txt 2>&1 - - if ! grep -q 'Failed to test access' output.txt; then - printf "Expected registry validation to fail before installation, but did not.\n\n" - cat output.txt - exit 1 - fi - - rm output.txt - - ./bin/kots upstream upgrade not-an-app \ - --airgap-bundle not-an-app.airgap \ - --kotsadm-registry localhost:1234/not-a-namespace \ - --registry-username not-a-username \ - --registry-password not-a-password \ - --namespace not-a-namespace > output.txt 2>&1 - - if ! grep -q 'Failed to test access' output.txt; then - printf "Expected registry validation to fail before upgrading, but did not.\n\n" - cat output.txt - exit 1 - fi - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-native-helm-v2: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0} - ] - env: - APP_SLUG: native-helm-v2 - INITIAL_VERSION: '0.1.1' - UPGRADE_VERSION: '0.2.1' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: azure/setup-helm@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$APP_SLUG" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - - name: run the test - run: | - set +e - echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} | base64 -d > license.yaml - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --app-version-label "$INITIAL_VERSION" \ - --no-port-forward \ - --namespace "$APP_SLUG" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$APP_SLUG" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit $EXIT_CODE - fi - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly - - if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-chart-release; then - printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG" - helm ls -n "$APP_SLUG" - exit 1 - fi - - if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-other-chart-release; then - printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG" - helm ls -n "$APP_SLUG" - exit 1 - fi - - # validate that `helm get values` works for the v1beta2 chart - - if ! helm get values my-chart-release -n "$APP_SLUG" | grep -q my-value; then - printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG" - helm get values my-chart-release -n "$APP_SLUG" - exit 1 - fi - - # upgrade the app version - - ./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy - - # wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2 - - COUNTER=1 - while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-chart-release)" != "" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for my-chart-release to be uninstalled" - helm ls -n "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - COUNTER=1 - while [ "$(helm ls -n "$APP_SLUG" | grep my-other-chart-release | awk 'NR>0{print $3}')" != "2" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2" - helm ls -n "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # validate that `helm get values` works for the upgraded v1beta2 chart - if ! helm get values my-other-chart-release -n "$APP_SLUG" | grep -q my-value; then - printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG" - helm get values my-chart-release -n "$APP_SLUG" - exit 1 - fi - - # wait for the app to be ready again - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$APP_SLUG" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # validate that the conditional chart is installed - COUNTER=1 - while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" == "" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for my-conditional-chart-release to be installed" - helm ls -n "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - # toggle the config option to exclude the conditional chart - ./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG" - - # wait for my-conditional-chart-release to be uninstalled - COUNTER=1 - while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" != "" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for my-conditional-chart-release to be uninstalled" - helm ls -n "$APP_SLUG" - exit 1 - fi - sleep 1 - done - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-deployment-orchestration: + # validate-existing-online-install-minimal: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: | + # docker load -i e2e/bin/e2e-deps.tar + # chmod +x e2e/bin/* + # chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # kots-namespace: 'qakotsregression' + # test-focus: 'Regression' + # k8s-distribution: k3s + # k8s-version: v1.26 + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-smoke-test: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + # continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'Smoke Test' + # kots-namespace: 'smoke-test' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-minimal-rbac: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0}, + # {distribution: openshift, version: 4.13.0-okd} + # ] + # env: + # APP_SLUG: minimal-rbac + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h | tee output.txt + + # if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + # echo "Expected to see an RBAC error for preflight checks, but did not" + # exit 1 + # fi + + # if ! grep -q 'The app was not deployed.' output.txt; then + # printf "Expected to see message about app not being deployed, but did not\n" + # exit 1 + # fi + + # if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + # printf "Did not expect to see a failure about number of nodes, but did\n" + # exit 1 + # fi + + # if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm role not found in namespace $APP_SLUG" + # kubectl get role -n "$APP_SLUG" + # exit 1 + # fi + + # if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm rolebinding not found in namespace $APP_SLUG" + # kubectl get rolebinding -n "$APP_SLUG" + # exit 1 + # fi + + # if kubectl get clusterrole | grep -q kotsadm; then + # echo "found kotsadm clusterrole in minimal RBAC install" + # kubectl get clusterrole + # exit 1 + # fi + + # if kubectl get clusterrolebinding | grep -q kotsadm; then + # echo "found kotsadm clusterrolebinding in minimal RBAC install" + # kubectl get clusterrolebinding + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-backup-and-restore: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + # continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'Backup and Restore' + # kots-namespace: 'backup-and-restore' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-no-required-config: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'No Required Config' + # kots-namespace: 'no-required-config' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-strict-preflight-checks: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: strict-preflight-checks + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --skip-preflights \ + # --kotsadm-tag 24h | tee output.txt + + # if ! grep -q 'The app was not deployed.' output.txt; then + # printf "Expected to see message about app not being deployed, but did not\n" + # exit 1 + # fi + + # if ! grep FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then + # printf "Expected to see a failure about kubernetes version, but did not\n" + # exit 1 + # fi + + # if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then + # echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs" + # echo "kotsadm logs:" + # kubectl logs deploy/kotsadm -n "$APP_SLUG" + # exit 1 + # fi + + # # disable the strict preflight check and app should deploy successfully + # ./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # printf "App is installed successfully and is ready\n\n" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-config: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'Config Validation' + # kots-namespace: 'config-validation' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-version-history-pagination: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: k3s-local, version: v1.27.1-k3s1} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'Version History Pagination' + # kots-namespace: 'version-history-pagination' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-change-license: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'Change License' + # kots-namespace: 'change-license' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-minimal-rbac-override: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0}, + # {distribution: openshift, version: 4.13.0-okd} + # ] + # env: + # APP_SLUG: minimal-rbac + # APP_VERSION_LABEL: "0.0.1" + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: minimal rbac override on command line + # run: | + # set +e + # echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --app-version-label "$APP_VERSION_LABEL" \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h \ + # --use-minimal-rbac | tee output.txt + + # if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + # echo "Expected to see an RBAC error for preflight checks, but did not" + # exit 1 + # fi + + # if ! grep -q 'The app was not deployed.' output.txt; then + # printf "Expected to see message about app not being deployed, but did not\n" + # exit 1 + # fi + + # if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + # printf "Did not expect to see a failure about number of nodes, but did\n" + # exit 1 + # fi + + # if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm role not found in namespace $APP_SLUG" + # kubectl get role -n "$APP_SLUG" + # exit 1 + # fi + + # if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm rolebinding not found in namespace $APP_SLUG" + # kubectl get rolebinding -n "$APP_SLUG" + # exit 1 + # fi + + # if kubectl get clusterrole | grep -q kotsadm; then + # echo "found kotsadm clusterrole in minimal RBAC install" + # kubectl get clusterrole + # exit 1 + # fi + + # if kubectl get clusterrolebinding | grep -q kotsadm; then + # echo "found kotsadm clusterrolebinding in minimal RBAC install" + # kubectl get clusterrolebinding + # exit 1 + # fi + + # - name: create namespace and dockerhub secret + # run: | + # kubectl delete ns "$APP_SLUG" --ignore-not-found + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: no minimal rbac override on command line + # run: | + # set +e + # echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --app-version-label "$APP_VERSION_LABEL" \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h | tee output.txt + + # if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + # echo "Did not expect to see an RBAC error for preflight checks, but did" + # exit 1 + # fi + + # if ! grep -q 'The app was not deployed.' output.txt; then + # printf "Expected to see message about app not being deployed, but did not\n" + # exit 1 + # fi + + # if ! grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + # printf "Expected to see a failure about number of nodes, but did not\n" + # exit 1 + # fi + + # if kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm role found in cluster scoped install" + # kubectl get role -n "$APP_SLUG" + # exit 1 + # fi + + # if kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + # echo "kotsadm rolebinding found in cluster scoped install" + # kubectl get rolebinding -n "$APP_SLUG" + # exit 1 + # fi + + # if ! kubectl get clusterrole | grep -q kotsadm; then + # echo "No kotsadm clusterrole found in cluster scoped install" + # kubectl get clusterrole + # exit 1 + # fi + + # if ! kubectl get clusterrolebinding | grep -q kotsadm; then + # echo "No kotsadm clusterrolebinding found in cluster scoped install" + # kubectl get clusterrolebinding + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-multi-namespace: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + # continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + # env: + # APP_SLUG: multi-namespace-yeti + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - uses: azure/setup-helm@v3 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 180 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly + + # if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then + # printf "postgresql helm release not found in postgres-test namespace\n\n" + # helm ls -n postgres-test + # exit 1 + # fi + + # if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then + # printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-kots-pull: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + # continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + # env: + # APP_NAME: multi-namespace-yeti + # APP_SLUG: multi-namespace + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_NAME" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_NAME" + + # - name: run kots pull + # run: | + # set +e + # echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml + # ./bin/kots pull "$APP_NAME/automated" \ + # --license-file license.yaml \ + # --shared-password password \ + # --namespace "$APP_NAME" \ + # --exclude-admin-console + + # kubectl create ns "$APP_NAME" + # kubectl create ns nginx-test + # kubectl create ns redis-test + # kubectl create ns postgres-test + + # # HACK: without operator, additonal namespaces don't get image pull secrets + # echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} | base64 -d > replicated-registry-auth.json + # kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json + # kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json + # kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json + # kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json + # kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json + + # kustomize build "$PWD/$APP_SLUG/overlays/midstream" | kubectl apply -f - + # kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" | kubectl apply -f - + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "Failed to apply spec" + # echo "------pods:" + # kubectl get pods -A + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # echo "Waiting for pods to start" + + # COUNTER=1 + # while [ "$(kubectl get pods --no-headers | grep -v Running | grep -cv Completed)" -gt 0 ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for pods to start" + # kubectl get pods -A + # exit 1 + # fi + # sleep 1 + # done + + # echo "All pods started" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_NAME" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-app-version-label: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: app-version-label + # APP_VERSION_LABEL: v1.0.0 + # LATEST_APP_VERSION_LABEL: v1.0.1 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: test kots install with version label + # run: | + # set +e + # echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --app-version-label "$APP_VERSION_LABEL" \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # # test setting DockerHub credentials + + # set +e + + # # TODO: deploy and check secrets are actually created and images are pulled + # ./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG" + # ./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG" + # if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then + # echo "Found DockerHub secret in ${APP_SLUG} latest version" + # else + # echo "No DockerHub secret found in appication namespace" + # exit 1 + # fi + + # - name: remove the app + # run: | + # set +e + # ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force + + # - name: test kots install without version label + # run: | + # set +e + # echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-helm-install-order: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: helm-install-order + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --skip-preflights \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # printf "App is installed successfully and is ready\n\n" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-no-redeploy-on-restart: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: no-redeploy-on-restart + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --skip-preflights \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # # wait for application job to be created + # COUNTER=1 + # while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" == "0" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 60 ]; then + # echo "Timed out waiting for job to be created" + # exit 1 + # fi + # sleep 1 + # done + + # # delete the application job and restart the admin console + # kubectl delete jobs -n "$APP_SLUG" --all + # kubectl delete pods -l app=kotsadm -n "$APP_SLUG" + + # # wait for old kotsadm pod to terminate + # COUNTER=1 + # while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" | awk 'NR>1' | wc -l)" != "1" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 60 ]; then + # echo "More than 1 kotsadm pod found" + # exit 1 + # fi + # sleep 1 + # done + + # # wait for new kotsadm pod to become ready + # kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s + + # # delay in case the app takes a bit to be deployed + # sleep 20 + + # # validate that the application wasn't re-deployed and the job wasn't re-created + # if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" != "0" ]; then + # echo "App should not be re-deployed after restart" + # exit 1 + # fi + + # printf "Success. App was not re-deployed after restart\n\n" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-kubernetes-installer-preflight: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: kubernetes-installer-preflight + # steps: + # - uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + + # echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} | base64 -d > license.yaml + + # # Fake kurl installation using the crd and installer spec from the application manifests: + + # # Pull application manifests + # ./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password + + # # Apply installer crd + # kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml" + + # # Wait for crd to be created + # kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh + + # # Seems that the above does not always guarantee the crd exists? So just in case... + # sleep 10 + + # # Apply installer + # kubectl apply -f "$APP_SLUG/upstream/installer.yaml" + + # # Create kurl-config configmap in kube-system + # kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094 + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # # validate that preflight checks ran + # JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}" + # if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then + # echo "Preflight checks did not run" + # exit 1 + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # try get apps without namespace (using kubeconfig) + # # validate that output is the same as above + # mkdir -p /tmp/.kube + # sudo cp "$KUBECONFIG" /tmp/.kube/config + # sudo chmod -R 777 /tmp/.kube + # export KUBECONFIG=/tmp/.kube/config + # kubectl config set-context --current --namespace="$APP_SLUG" + # if [ "$(./bin/kots get apps | awk 'NR>1{print $2}')" != "ready" ]; then + # echo "kots get apps output is not the same as above" + # exit 1 + # fi + + # printf "App is installed successfully and is ready\n\n" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-kots-push-images-anonymous: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: run kots admin-console push-images + # run: | + # set +e + # ./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }} + + + # validate-kots-admin-console-generate-manifests: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: minimal-rbac + # BASE_KOTS_VERSION: v1.72.0 + # NAMESPACE: generate-manifests + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: run kots admin-console generate-manifests without k8s context + # run: | + # set +e + # ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password + + # - name: validate that ./admin-console exists and is not empty + # run: | + # set +e + # if [ ! -d ./admin-console ]; then + # echo "admin-console directory does not exist" + # exit 1 + # fi + # if [ -z "$(ls -A ./admin-console)" ]; then + # echo "admin-console directory is empty" + # exit 1 + # fi + + # - name: remove admin-console directory + # run: rm -rf ./admin-console + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download base kots version + # run: | + # curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ + # && tar zxvf kots_linux_amd64.tar.gz \ + # && mv kots "kots-$BASE_KOTS_VERSION" + + # - name: create namespace + # run: | + # set +e + # kubectl create namespace "$NAMESPACE" + + # - name: run kots admin-console generate-manifests using base kots version with k8s context + # run: | + # set +e + # "./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password + + # - name: apply the generated manifests + # run: | + # set +e + # kubectl apply -f ./admin-console -n "$NAMESPACE" + + # - name: wait for the kotsadm-minio-0 pod to be created + # run: | + # set +e + # COUNTER=1 + # while ! kubectl get pods -n "$NAMESPACE" | grep -q kotsadm-minio-0; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 30 ]; then + # echo "timed out waiting for kotsadm-minio-0 pod to be created" + # exit 1 + # fi + # sleep 1 + # done + + # - name: wait for kotsadm-minio-0 pod to be ready + # run: | + # set +e + # kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" + + # - name: wait for the kotsadm deployment to be ready + # run: | + # set +e + # kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" + + # - name: run kots admin-console generate-manifests using new kots version with k8s context + # run: | + # set +e + # ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # - name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed + # run: | + # set +e + # if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then + # echo "admin-console/minio-statefulset.yaml does not have initContainers" + # exit 1 + # fi + + # - name: apply the generated manifests + # run: | + # set +e + # kubectl apply -f ./admin-console -n "$NAMESPACE" + + # - name: wait for kotsadm-minio-0 pod to be ready + # run: | + # set +e + # sleep 10 + # kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" + + # - name: wait for the kotsadm deployment to be ready + # run: | + # set +e + # kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" + + # - name: run kots admin-console generate-manifests using new kots version with k8s context + # run: | + # set +e + # ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # - name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed + # run: | + # set +e + # if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then + # echo "admin-console/minio-statefulset.yaml has initContainers" + # exit 1 + # fi + + # - name: apply the generated manifests + # run: | + # set +e + # kubectl apply -f ./admin-console -n "$NAMESPACE" + + # - name: wait for kotsadm-minio-0 pod to be ready + # run: | + # set +e + # sleep 10 + # kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE" + + # - name: wait for the kotsadm deployment to be ready + # run: | + # set +e + # kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE" + + # - name: print pods and logs on failure + # if: failure() + # run: | + # echo "------pods:" + # kubectl -n "$NAMESPACE" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE" + # echo "------kotsadm-minio logs" + # kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: automated-${{ github.run_id }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-min-kots-version: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] + # env: + # APP_SLUG: min-kots-version + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: | + # docker load -i e2e/bin/e2e-deps.tar + # chmod +x e2e/bin/* + # chmod +x bin/* + # cp ./bin/kots /usr/local/bin/kubectl-kots + # sudo apt-get update -y && sudo apt-get install jq -y + # - uses: ./.github/actions/kots-e2e + # id: kots-e2e + # with: + # test-focus: 'Min KOTS Version' + # kots-namespace: 'min-kots-version' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # kotsadm-image-registry: ttl.sh + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + # k8s-cluster-skip-teardown: true + + # - name: validate that kots install fails early + # run: | + # set +e + + # result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null) + # echo "$result" + + # if [[ "$result" == *"requires"* ]] && [[ "$result" == *"10000.0.0"* ]]; then + # exit 0 + # else + # exit 1 + # fi + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} + + + # validate-target-kots-version: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: | + # docker load -i e2e/bin/e2e-deps.tar + # chmod +x e2e/bin/* + # chmod +x bin/* + # cp ./bin/kots /usr/local/bin/kubectl-kots + # sudo apt-get update -y && sudo apt-get install jq -y + # - uses: ./.github/actions/kots-e2e + # id: kots-e2e + # with: + # test-focus: 'Target KOTS Version' + # kots-namespace: 'target-kots-version' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + # k8s-cluster-skip-teardown: true + + # - name: validate that kots install fails early + # run: | + # set +e + + # result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null) + # echo "$result" + + # if [[ "$result" == *"requires"* ]] && [[ "$result" == *"1.0.0"* ]]; then + # exit 0 + # else + # exit 1 + # fi + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} + + + # validate-range-kots-version: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: | + # docker load -i e2e/bin/e2e-deps.tar + # chmod +x e2e/bin/* + # chmod +x bin/* + # cp ./bin/kots /usr/local/bin/kubectl-kots + # sudo apt-get update -y && sudo apt-get install jq -y + # - uses: ./.github/actions/kots-e2e + # id: kots-e2e + # with: + # test-focus: 'Range KOTS Version' + # kots-namespace: 'range-kots-version' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + # k8s-cluster-skip-teardown: true + + # - name: validate that kots install fails early + # run: | + # set +e + + # result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null) + # echo "$result" + + # if [[ "$result" == *"requires"* ]] && [[ "$result" == *"11000.0.0"* ]]; then + # exit 0 + # else + # exit 1 + # fi + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }} + + + # validate-kots-upgrade: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} + # continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + # env: + # APP_SLUG: postgres-to-rqlite + # BASE_KOTS_VERSION: v1.57.0 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }} + # export-kubeconfig: true + + # - name: download base kots version + # run: | + # curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ + # && tar zxvf kots_linux_amd64.tar.gz \ + # && mv kots "kots-$BASE_KOTS_VERSION" + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml + + # # install using the base KOTS version + + # "./kots-$BASE_KOTS_VERSION" \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --port-forward=false \ + # --namespace "$APP_SLUG" \ + # --shared-password password + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # upgrade using the new KOTS version + + # ./bin/kots admin-console upgrade \ + # --namespace "$APP_SLUG" \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # # verify that the postgres to rqlite migration was successful + + # if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then + # echo "Failed to find a successful migration log line" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG" + # exit 1 + # fi + + # # verify that the minio migration happened + + # if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then + # echo "Failed to find initContainers in the kotsadm-minio statefulset" + # echo "kotsadm-minio statefulset:" + # kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml + # exit 1 + # fi + + # # make sure app is still installed and ready + + # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then + # echo "App is not ready after the upgrade" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # printf "App is still installed and is ready after the migration\n\n" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-kots-helm-release-secret-migration: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: helm-release-secret-migration + # RELEASE_NAME: helm-release-chart + # RELEASE_NAMESPACE: helm-release + # BASE_KOTS_VERSION: v1.94.0 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download base kots version + # run: | + # curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \ + # && tar zxvf kots_linux_amd64.tar.gz \ + # && mv kots "kots-$BASE_KOTS_VERSION" + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} | base64 -d > license.yaml + + # # install using the base KOTS version + # "./kots-$BASE_KOTS_VERSION" \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward=true \ + # --namespace "$APP_SLUG" \ + # --shared-password password + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # verify that the helm release secret is created in the kotsadm namespace + # releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}') + # if [ -z "$releaseSecretName" ]; then + # echo "Failed to find the helm release secret in the $APP_SLUG namespace" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # # if there are more than one helm release secrets, fail + # if [ "$(echo "$releaseSecretName" | wc -l)" -gt 1 ]; then + # echo "Found more than one helm release secret in the $APP_SLUG namespace" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # # upgrade using the new KOTS version + # ./bin/kots admin-console upgrade \ + # --namespace "$APP_SLUG" \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # # make a config change and redeploy the app + # ./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG" + + # # make sure app is still installed and ready + # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then + # echo "App is not ready after the upgrade" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # COUNTER=1 + # while [ "$(helm ls -n "$RELEASE_NAMESPACE" | grep "$RELEASE_NAME" | awk 'NR>0{print $3}')" != "2" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2" + # helm ls -n "$RELEASE_NAMESPACE" + # exit 1 + # fi + # sleep 1 + # done + + # # verify that the helm release secret is created in the helm relase namespace + # COUNT=1 + # releaseSecret="" + # while [ -z "$releaseSecret" ]; do + # ((COUNT += 1)) + # if [ $COUNT -gt 10 ]; then + # echo "Timed out waiting for the helm release secret to be migrated" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE") + # done + + # # verify that the release secret in app namepspace is deleted + # oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG") + # if [ -n "$oldReleaseSecret" ]; then + # echo "Found the helm release secret in the $APP_SLUG namespace" + # echo "$oldReleaseSecret" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # # verify that there are two helm release secrets in the helm release namespace + # releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}') + # releaseSecretNamesCount=$(echo "$releaseSecretNames" | wc -w) + # if [ "$releaseSecretNamesCount" -ne 2 ]; then + # echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)" + # kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + + # printf "Helm release secret migration test passed\n\n" + # ./bin/kots get apps --namespace "$APP_SLUG" + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-multi-app-backup-and-restore: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'multi-app-backup-and-restore' + # kots-namespace: 'multi-app-backup-and-restore' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-multi-app-install: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'multi-app-install' + # kots-namespace: 'multi-app-install' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-airgap-smoke-test: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: download e2e deps + # uses: actions/download-artifact@v3 + # with: + # name: e2e + # path: e2e/bin/ + # - run: docker load -i e2e/bin/e2e-deps.tar + # - run: chmod +x e2e/bin/* + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + # - run: chmod +x bin/* + # - uses: ./.github/actions/kots-e2e + # with: + # test-focus: 'airgap-smoke-test' + # kots-namespace: 'airgap-smoke-test' + # k8s-distribution: ${{ matrix.cluster.distribution }} + # k8s-version: ${{ matrix.cluster.version }} + # testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' + # testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + # kots-airgap: true + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + # replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' + # kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' + # kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + + # validate-remove-app: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0, instance-type: r1.medium} + # ] + # env: + # APP_SLUG: remove-app + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # instance-type: ${{ matrix.cluster.instance-type }} + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + + # echo ${{ secrets.REMOVE_APP_LICENSE }} | base64 -d > license.yaml + # HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG") + # APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG") + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # test that --undeploy deletes application resources from the cluster + + # ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy + + # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then + # printf "Application resources are still found in the cluster\n\n" + # kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" + # exit 1 + # fi + + # if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then + # printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n" + # kubectl get crd rabbitmqclusters.rabbitmq.com + # exit 1 + # fi + + # for ns in "${HELM_APP_NAMESPACES[@]}"; do + # if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then + # printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns" + # kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 + # exit 1 + # fi + # done + + # # re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force + + # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # for ns in "${APP_NAMESPACES[@]}"; do + # if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then + # printf "%s namespace does not contain any resources\n\n" "$ns" + # kubectl get all,secrets,configmap,pvc -n "$ns" + # exit 1 + # fi + # done + + # if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then + # printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n" + # kubectl get crd rabbitmqclusters.rabbitmq.com + # exit 1 + # fi + + # if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then + # printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n" + # kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system + # exit 1 + # fi + + # for ns in "${HELM_APP_NAMESPACES[@]}"; do + # if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then + # printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns" + # kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 + # exit 1 + # fi + # done + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-registry-check: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: run the test + # run: | + # set +e + + # ./bin/kots admin-console push-images \ + # not-an-app.airgap localhost:1234/not-a-namespace \ + # --registry-username not-a-username \ + # --registry-password not-a-password > output.txt 2>&1 + + # if ! grep -q 'Failed to test access' output.txt; then + # printf "Expected registry validation to fail before pushing images, but did not.\n\n" + # cat output.txt + # exit 1 + # fi + + # rm output.txt + + # ./bin/kots install not-an-app \ + # --airgap-bundle not-an-app.airgap \ + # --kotsadm-registry localhost:1234/not-a-namespace \ + # --registry-username not-a-username \ + # --registry-password not-a-password > output.txt 2>&1 + + # if ! grep -q 'Failed to test access' output.txt; then + # printf "Expected registry validation to fail before installation, but did not.\n\n" + # cat output.txt + # exit 1 + # fi + + # rm output.txt + + # ./bin/kots upstream upgrade not-an-app \ + # --airgap-bundle not-an-app.airgap \ + # --kotsadm-registry localhost:1234/not-a-namespace \ + # --registry-username not-a-username \ + # --registry-password not-a-password \ + # --namespace not-a-namespace > output.txt 2>&1 + + # if ! grep -q 'Failed to test access' output.txt; then + # printf "Expected registry validation to fail before upgrading, but did not.\n\n" + # cat output.txt + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-native-helm-v2: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: native-helm-v2 + # INITIAL_VERSION: '0.1.1' + # UPGRADE_VERSION: '0.2.1' + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - uses: azure/setup-helm@v3 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --app-version-label "$INITIAL_VERSION" \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly + + # if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-chart-release; then + # printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + + # if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-other-chart-release; then + # printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + + # # validate that `helm get values` works for the v1beta2 chart + + # if ! helm get values my-chart-release -n "$APP_SLUG" | grep -q my-value; then + # printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG" + # helm get values my-chart-release -n "$APP_SLUG" + # exit 1 + # fi + + # # upgrade the app version + + # ./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy + + # # wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2 + + # COUNTER=1 + # while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-chart-release)" != "" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for my-chart-release to be uninstalled" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # COUNTER=1 + # while [ "$(helm ls -n "$APP_SLUG" | grep my-other-chart-release | awk 'NR>0{print $3}')" != "2" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # validate that `helm get values` works for the upgraded v1beta2 chart + # if ! helm get values my-other-chart-release -n "$APP_SLUG" | grep -q my-value; then + # printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG" + # helm get values my-chart-release -n "$APP_SLUG" + # exit 1 + # fi + + # # wait for the app to be ready again + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # validate that the conditional chart is installed + # COUNTER=1 + # while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" == "" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for my-conditional-chart-release to be installed" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # toggle the config option to exclude the conditional chart + # ./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG" + + # # wait for my-conditional-chart-release to be uninstalled + # COUNTER=1 + # while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" != "" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for my-conditional-chart-release to be uninstalled" + # helm ls -n "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-deployment-orchestration: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0} + # ] + # env: + # APP_SLUG: deployment-orchestration + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - uses: azure/setup-helm@v3 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$APP_SLUG" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + # - name: run the test + # run: | + # set +e + # echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} | base64 -d > license.yaml + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$APP_SLUG" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$APP_SLUG" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit $EXIT_CODE + # fi + + # function wait_for_log { + # local counter=1 + # local timeout=30 + # local log_pattern="$1" + + # while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "$log_pattern"; do + # ((counter += 1)) + # if [ $counter -gt $timeout ]; then + # echo "Timed out waiting for log $log_pattern" + # exit 1 + # fi + # sleep 1 + # done + # } + + # wait_for_log "applying phase -9999" + + # wait_for_log "applying phase -3" + # wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready" + + # wait_for_log "applying phase -2" + + # # validate that phase -1 has not deployed yet since we're waiting on the CR status fields + # if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "applying phase -1"; then + # printf "phase -1 was deployed before phase -2 completed" + # kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" + # exit 1 + # fi + + # wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true" + + # # set .status.tasks.extract=true + # kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge + + # wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true" + + # # set .status.tasks.transform=true + # kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge + + # wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true" + + # # set .status.tasks.load=true + # kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge + + # # now validate that the remaining phases are deployed (-1, 0, and 1) + # wait_for_log "applying phase -1" + # wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready" + + # wait_for_log "applying phase 0" + # wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready" + + # wait_for_log "applying phase 1" + # wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready" + + # # wait for the app to be ready + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$APP_SLUG" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG" + # exit 1 + # fi + # sleep 1 + # done + + # # remove the app + # ./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy + + # wait_for_log "deleting resources in phase -1" + # wait_for_log "deleting resources in phase 0" + # wait_for_log "deleting resources in phase 1" + # wait_for_log "deleting resources in phase 2" + # wait_for_log "deleting resources in phase 3" + + # # validate that the app reference was removed + # if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + + # validate-replicated-sdk: + # runs-on: ubuntu-20.04 + # needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] + # strategy: + # fail-fast: false + # matrix: + # cluster: [ + # {distribution: kind, version: v1.28.0}, + # {distribution: openshift, version: 4.13.0-okd} + # ] + # env: + # KOTS_NAMESPACE: replicated-sdk + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Create Cluster + # id: create-cluster + # uses: replicatedhq/replicated-actions/create-cluster@v1 + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # kubernetes-distribution: ${{ matrix.cluster.distribution }} + # kubernetes-version: ${{ matrix.cluster.version }} + # cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + # timeout-minutes: '120' + # ttl: 2h + # export-kubeconfig: true + + # - name: download kots binary + # uses: actions/download-artifact@v3 + # with: + # name: kots + # path: bin/ + + # - run: chmod +x bin/kots + + # - name: create namespace and dockerhub secret + # run: | + # kubectl create ns "$KOTS_NAMESPACE" + # kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$KOTS_NAMESPACE" + + # - name: run upgrade-to-replicated-sdk test + # id: upgrade-to-replicated-sdk + # env: + # APP_SLUG: upgrade-to-replicated-sdk + # run: | + # set +e + + # echo ${{ secrets.UPGRADE_TO_REPLICATED_SDK_LICENSE }} | base64 -d > license.yaml + # LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" + # echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$KOTS_NAMESPACE" \ + # --shared-password password \ + # --app-version-label v1.0.0 \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$KOTS_NAMESPACE" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit $EXIT_CODE + # fi + + # # wait for the app to be ready + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$KOTS_NAMESPACE" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit 1 + # fi + # sleep 1 + # done + + # # upgrade the app to the new version + # ./bin/kots upstream upgrade "$APP_SLUG" -n "$KOTS_NAMESPACE" --deploy + # sleep 5 + + # # wait for the app to be ready again + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$KOTS_NAMESPACE" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit 1 + # fi + # sleep 1 + # done + + # # get the version of the replicated-sdk that is running and set it as an output + # REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') + # echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" + + # - run: rm -rf ./replicated-sdk + # - name: Checkout replicated-sdk + # uses: actions/checkout@v4 + # with: + # repository: replicatedhq/replicated-sdk + # path: replicated-sdk + # ref: ${{ steps.upgrade-to-replicated-sdk.outputs.replicated-sdk-version }} + + # - name: Validate endpoints + # uses: ./replicated-sdk/.github/actions/validate-endpoints + # with: + # license-id: ${{ steps.upgrade-to-replicated-sdk.outputs.license-id }} + # license-fields: '[{"name":"expires_at","value": ""}]' + # integration-enabled: true + # namespace: "$KOTS_NAMESPACE" + + # - name: remove upgrade-to-replicated-sdk app + # env: + # APP_SLUG: upgrade-to-replicated-sdk + # run: | + # # remove the app + # ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy + + # # validate that the app reference was removed + # if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # - name: run replicated-sdk-subchart-native-helm-v1 test + # id: replicated-sdk-subchart-native-helm-v1 + # env: + # APP_SLUG: replicated-sdk-subchart-native-helm-v1 + # run: | + # set +e + + # echo ${{ secrets.REPLICATED_SDK_SUBCHART_NATIVE_HELM_V1 }} | base64 -d > license.yaml + # LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" + # echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$KOTS_NAMESPACE" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$KOTS_NAMESPACE" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit $EXIT_CODE + # fi + + # # wait for the app to be ready + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$KOTS_NAMESPACE" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit 1 + # fi + # sleep 1 + # done + + # # get the version of the replicated-sdk that is running and set it as an output + # REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') + # echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" + + # - run: rm -rf ./replicated-sdk + # - name: Checkout replicated-sdk + # uses: actions/checkout@v4 + # with: + # repository: replicatedhq/replicated-sdk + # path: replicated-sdk + # ref: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.replicated-sdk-version }} + + # - name: Validate endpoints + # uses: ./replicated-sdk/.github/actions/validate-endpoints + # with: + # license-id: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.license-id }} + # license-fields: '[{"name":"expires_at","value": ""}]' + # integration-enabled: true + # namespace: "$KOTS_NAMESPACE" + + # - name: remove replicated-sdk-subchart-native-helm-v1 app + # env: + # APP_SLUG: replicated-sdk-subchart-native-helm-v1 + # run: | + # # remove the app + # ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy + + # # validate that the app reference was removed + # if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # - name: run replicated-sdk-subchart-replicated-helm test + # id: replicated-sdk-subchart-replicated-helm + # env: + # APP_SLUG: replicated-sdk-subchart-replicated-helm + # run: | + # set +e + + # echo ${{ secrets.REPLICATED_SDK_SUBCHART_REPLICATED_HELM }} | base64 -d > license.yaml + # LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" + # echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" + + # ./bin/kots \ + # install "$APP_SLUG/automated" \ + # --license-file license.yaml \ + # --no-port-forward \ + # --namespace "$KOTS_NAMESPACE" \ + # --shared-password password \ + # --kotsadm-registry ttl.sh \ + # --kotsadm-namespace automated-${{ github.run_id }} \ + # --kotsadm-tag 24h + + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "------pods:" + # kubectl -n "$KOTS_NAMESPACE" get pods + # echo "------kotsadm logs" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit $EXIT_CODE + # fi + + # # wait for the app to be ready + # COUNTER=1 + # while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do + # ((COUNTER += 1)) + # if [ $COUNTER -gt 120 ]; then + # echo "Timed out waiting for app to be ready" + # ./bin/kots get apps --namespace "$KOTS_NAMESPACE" + # echo "kotsadm logs:" + # kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" + # exit 1 + # fi + # sleep 1 + # done + + # # get the version of the replicated-sdk that is running and set it as an output + # REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') + # echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" + + # - run: rm -rf ./replicated-sdk + # - name: Checkout replicated-sdk + # uses: actions/checkout@v4 + # with: + # repository: replicatedhq/replicated-sdk + # path: replicated-sdk + # ref: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.replicated-sdk-version }} + + # - name: Validate endpoints + # uses: ./replicated-sdk/.github/actions/validate-endpoints + # with: + # license-id: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.license-id }} + # license-fields: '[{"name":"expires_at","value": ""}]' + # integration-enabled: true + # namespace: "$KOTS_NAMESPACE" + # deployed-via-kubectl: true + + # - name: remove replicated-sdk-subchart-replicated-helm app + # env: + # APP_SLUG: replicated-sdk-subchart-replicated-helm + # run: | + # # remove the app + # ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy + + # # validate that the app reference was removed + # if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then + # printf "App reference was not removed\n\n" + # exit 1 + # fi + + # - name: Generate support bundle on failure + # if: failure() + # uses: ./.github/actions/generate-support-bundle + # with: + # kots-namespace: "$APP_SLUG" + # aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' + # aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' + + # - name: Remove Cluster + # id: remove-cluster + # uses: replicatedhq/replicated-actions/remove-cluster@v1 + # if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + # continue-on-error: true + # with: + # api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + # cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + + validate-support-bundle: runs-on: ubuntu-20.04 needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] strategy: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.28.0} - ] + {distribution: kind, version: v1.28.0}, + {distribution: openshift, version: 4.13.0-okd} + ] env: - APP_SLUG: deployment-orchestration + APP_SLUG: kotsadm-smoke-test steps: - name: Checkout uses: actions/checkout@v4 - - uses: azure/setup-helm@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} + - name: Install jq + uses: dcarbone/install-jq-action@v1.0.1 + + - name: Download support-bundle binary + run: | + RELEASE="$( + curl -sfL https://api.github.com/repos/replicatedhq/troubleshoot/releases/latest | \ + grep '"tag_name":' | \ + sed -E 's/.*"(v[^"]+)".*/\1/' + )" + curl -fsLO "https://github.com/replicatedhq/troubleshoot/releases/download/${RELEASE}/support-bundle_linux_amd64.tar.gz" + tar xzf support-bundle_linux_amd64.tar.gz - name: Create Cluster id: create-cluster @@ -3416,10 +3903,12 @@ jobs: kubectl create ns "$APP_SLUG" kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" - - name: run the test + - name: install kots and the application run: | - set +e - echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} | base64 -d > license.yaml + set -e + set -o pipefail + + echo ${{ secrets.SMOKE_TEST_LICENSE }} | base64 -d > license.yaml ./bin/kots \ install "$APP_SLUG/automated" \ --license-file license.yaml \ @@ -3439,60 +3928,6 @@ jobs: exit $EXIT_CODE fi - function wait_for_log { - local counter=1 - local timeout=30 - local log_pattern="$1" - - while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "$log_pattern"; do - ((counter += 1)) - if [ $counter -gt $timeout ]; then - echo "Timed out waiting for log $log_pattern" - exit 1 - fi - sleep 1 - done - } - - wait_for_log "applying phase -9999" - - wait_for_log "applying phase -3" - wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready" - - wait_for_log "applying phase -2" - - # validate that phase -1 has not deployed yet since we're waiting on the CR status fields - if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "applying phase -1"; then - printf "phase -1 was deployed before phase -2 completed" - kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" - exit 1 - fi - - wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true" - - # set .status.tasks.extract=true - kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge - - wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true" - - # set .status.tasks.transform=true - kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge - - wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true" - - # set .status.tasks.load=true - kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge - - # now validate that the remaining phases are deployed (-1, 0, and 1) - wait_for_log "applying phase -1" - wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready" - - wait_for_log "applying phase 0" - wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready" - - wait_for_log "applying phase 1" - wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready" - # wait for the app to be ready COUNTER=1 while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do @@ -3506,329 +3941,40 @@ jobs: fi sleep 1 done - - # remove the app - ./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy - - wait_for_log "deleting resources in phase -1" - wait_for_log "deleting resources in phase 0" - wait_for_log "deleting resources in phase 1" - wait_for_log "deleting resources in phase 2" - wait_for_log "deleting resources in phase 3" - - # validate that the app reference was removed - if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" - exit 1 - fi - - - name: Generate support bundle on failure - if: failure() - uses: ./.github/actions/generate-support-bundle - with: - kots-namespace: "$APP_SLUG" - aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' - aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - - - name: Remove Cluster - id: remove-cluster - uses: replicatedhq/replicated-actions/remove-cluster@v1 - if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} - continue-on-error: true - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} - - - validate-replicated-sdk: - runs-on: ubuntu-20.04 - needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ] - strategy: - fail-fast: false - matrix: - cluster: [ - {distribution: kind, version: v1.28.0}, - {distribution: openshift, version: 4.13.0-okd} - ] - env: - KOTS_NAMESPACE: replicated-sdk - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Cluster - id: create-cluster - uses: replicatedhq/replicated-actions/create-cluster@v1 - with: - api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} - kubernetes-distribution: ${{ matrix.cluster.distribution }} - kubernetes-version: ${{ matrix.cluster.version }} - cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} - timeout-minutes: '120' - ttl: 2h - export-kubeconfig: true - - - name: download kots binary - uses: actions/download-artifact@v3 - with: - name: kots - path: bin/ - - - run: chmod +x bin/kots - - - name: create namespace and dockerhub secret - run: | - kubectl create ns "$KOTS_NAMESPACE" - kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$KOTS_NAMESPACE" - - - name: run upgrade-to-replicated-sdk test - id: upgrade-to-replicated-sdk - env: - APP_SLUG: upgrade-to-replicated-sdk - run: | - set +e - - echo ${{ secrets.UPGRADE_TO_REPLICATED_SDK_LICENSE }} | base64 -d > license.yaml - LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" - echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$KOTS_NAMESPACE" \ - --shared-password password \ - --app-version-label v1.0.0 \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$KOTS_NAMESPACE" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit $EXIT_CODE - fi - - # wait for the app to be ready - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$KOTS_NAMESPACE" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit 1 - fi - sleep 1 - done - - # upgrade the app to the new version - ./bin/kots upstream upgrade "$APP_SLUG" -n "$KOTS_NAMESPACE" --deploy + + # port forward to kotsadm + kill -9 $(sudo lsof -t -i:3000) 2>/dev/null || true + kubectl port-forward -n "$APP_SLUG" svc/kotsadm 3000:3000 & sleep 5 + + # get the support bundle command + KOTSADM_AUTHSTRING=$(kubectl get secret kotsadm-authstring -n $APP_SLUG -o jsonpath='{.data.kotsadm-authstring}' | base64 -d) + SUPPORT_BUNDLE_COMMAND=$(curl -sSL -X POST http://localhost:3000/api/v1/troubleshoot/app/$APP_SLUG/supportbundlecommand --data '{"origin": "http://localhost:3000"}' -H "Authorization: $KOTSADM_AUTHSTRING" | jq -r .command[1]) + SUPPORT_BUNDLE_COMMAND="./support-bundle ${SUPPORT_BUNDLE_COMMAND#kubectl support-bundle}" - # wait for the app to be ready again - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$KOTS_NAMESPACE" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit 1 - fi - sleep 1 - done - - # get the version of the replicated-sdk that is running and set it as an output - REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') - echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" - - - run: rm -rf ./replicated-sdk - - name: Checkout replicated-sdk - uses: actions/checkout@v4 - with: - repository: replicatedhq/replicated-sdk - path: replicated-sdk - ref: ${{ steps.upgrade-to-replicated-sdk.outputs.replicated-sdk-version }} - - - name: Validate endpoints - uses: ./replicated-sdk/.github/actions/validate-endpoints - with: - license-id: ${{ steps.upgrade-to-replicated-sdk.outputs.license-id }} - license-fields: '[{"name":"expires_at","value": ""}]' - integration-enabled: true - namespace: "$KOTS_NAMESPACE" - - - name: remove upgrade-to-replicated-sdk app - env: - APP_SLUG: upgrade-to-replicated-sdk - run: | - # remove the app - ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy - - # validate that the app reference was removed - if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" - exit 1 - fi - - - name: run replicated-sdk-subchart-native-helm-v1 test - id: replicated-sdk-subchart-native-helm-v1 - env: - APP_SLUG: replicated-sdk-subchart-native-helm-v1 - run: | - set +e - - echo ${{ secrets.REPLICATED_SDK_SUBCHART_NATIVE_HELM_V1 }} | base64 -d > license.yaml - LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" - echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$KOTS_NAMESPACE" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$KOTS_NAMESPACE" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit $EXIT_CODE - fi - - # wait for the app to be ready - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$KOTS_NAMESPACE" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit 1 - fi - sleep 1 - done - - # get the version of the replicated-sdk that is running and set it as an output - REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') - echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" - - - run: rm -rf ./replicated-sdk - - name: Checkout replicated-sdk - uses: actions/checkout@v4 - with: - repository: replicatedhq/replicated-sdk - path: replicated-sdk - ref: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.replicated-sdk-version }} - - - name: Validate endpoints - uses: ./replicated-sdk/.github/actions/validate-endpoints - with: - license-id: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.license-id }} - license-fields: '[{"name":"expires_at","value": ""}]' - integration-enabled: true - namespace: "$KOTS_NAMESPACE" - - - name: remove replicated-sdk-subchart-native-helm-v1 app - env: - APP_SLUG: replicated-sdk-subchart-native-helm-v1 - run: | - # remove the app - ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy - - # validate that the app reference was removed - if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" + # run support bundle command + $SUPPORT_BUNDLE_COMMAND --interactive=false + + # validate that the support bundle was generated and custom redactions were applied + tar xzf support-bundle-*.tar.gz + cat support-bundle-*/cluster-info/cluster_version.json | jq -r .info.major | grep -q "***HIDDEN***" + cat support-bundle-*/cluster-info/cluster_version.json | jq -r .info.minor | grep -q "***HIDDEN***" + + # validate that bundle was uploaded to admin console + BUNDLE_COUNT=$(curl -sSL -X GET http://localhost:3000/api/v1/troubleshoot/app/$APP_SLUG/supportbundles -H "Authorization: $KOTSADM_AUTHSTRING" | jq -r '.supportBundles | length') + if [ "$BUNDLE_COUNT" != "1" ]; then + echo "Expected 1 support bundle to be uploaded to admin console, found $BUNDLE_COUNT" exit 1 fi - - name: run replicated-sdk-subchart-replicated-helm test - id: replicated-sdk-subchart-replicated-helm - env: - APP_SLUG: replicated-sdk-subchart-replicated-helm - run: | - set +e - - echo ${{ secrets.REPLICATED_SDK_SUBCHART_REPLICATED_HELM }} | base64 -d > license.yaml - LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)" - echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT" - - ./bin/kots \ - install "$APP_SLUG/automated" \ - --license-file license.yaml \ - --no-port-forward \ - --namespace "$KOTS_NAMESPACE" \ - --shared-password password \ - --kotsadm-registry ttl.sh \ - --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "------pods:" - kubectl -n "$KOTS_NAMESPACE" get pods - echo "------kotsadm logs" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit $EXIT_CODE - fi - - # wait for the app to be ready - COUNTER=1 - while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do - ((COUNTER += 1)) - if [ $COUNTER -gt 120 ]; then - echo "Timed out waiting for app to be ready" - ./bin/kots get apps --namespace "$KOTS_NAMESPACE" - echo "kotsadm logs:" - kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE" - exit 1 - fi - sleep 1 - done - - # get the version of the replicated-sdk that is running and set it as an output - REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$') - echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT" - - - run: rm -rf ./replicated-sdk - - name: Checkout replicated-sdk - uses: actions/checkout@v4 - with: - repository: replicatedhq/replicated-sdk - path: replicated-sdk - ref: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.replicated-sdk-version }} - - - name: Validate endpoints - uses: ./replicated-sdk/.github/actions/validate-endpoints - with: - license-id: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.license-id }} - license-fields: '[{"name":"expires_at","value": ""}]' - integration-enabled: true - namespace: "$KOTS_NAMESPACE" - deployed-via-kubectl: true - - - name: remove replicated-sdk-subchart-replicated-helm app - env: - APP_SLUG: replicated-sdk-subchart-replicated-helm - run: | - # remove the app - ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy + # fetch the redactor report + BUNDLE_ID=$(curl -sSL -X GET http://localhost:3000/api/v1/troubleshoot/app/$APP_SLUG/supportbundles -H "Authorization: $KOTSADM_AUTHSTRING" | jq -r .supportBundles[0].id) + curl -sSL -X GET http://localhost:3000/api/v1/troubleshoot/supportbundle/$BUNDLE_ID/redactions > redactions.json - # validate that the app reference was removed - if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then - printf "App reference was not removed\n\n" + # validate that default ip address redaction was applied + IP_ADDRESS_REDACTION_COUNT=$(cat redactions.json | jq '.redactions.byRedactor."IP Addresses.regex.0" | length') + if [ "$IP_ADDRESS_REDACTION_COUNT" -eq "0" ]; then + echo "Expected IP address redaction count to be greater than 0" exit 1 fi @@ -3858,39 +4004,40 @@ jobs: - unit-test-web - vet-kots - ci-test-kots - # testim tests - - validate-existing-online-install-minimal - - validate-smoke-test - - validate-backup-and-restore - - validate-no-required-config - - validate-version-history-pagination - - validate-change-license - - validate-min-kots-version - - validate-target-kots-version - - validate-range-kots-version - - validate-multi-app-backup-and-restore - - validate-multi-app-install - - validate-airgap-smoke-test - - validate-config - # non-testim tests - - validate-minimal-rbac - - validate-minimal-rbac-override - - validate-multi-namespace - - validate-kots-pull - - validate-app-version-label - - validate-helm-install-order - - validate-no-redeploy-on-restart - - validate-kubernetes-installer-preflight - - validate-kots-upgrade - - validate-remove-app - - validate-registry-check - - validate-kots-helm-release-secret-migration - - validate-native-helm-v2 - - validate-deployment-orchestration - - validate-replicated-sdk - - validate-strict-preflight-checks - # cli-only tests - - validate-kots-push-images-anonymous + # # testim tests + # - validate-existing-online-install-minimal + # - validate-smoke-test + # - validate-backup-and-restore + # - validate-no-required-config + # - validate-version-history-pagination + # - validate-change-license + # - validate-min-kots-version + # - validate-target-kots-version + # - validate-range-kots-version + # - validate-multi-app-backup-and-restore + # - validate-multi-app-install + # - validate-airgap-smoke-test + # - validate-config + # # non-testim tests + # - validate-minimal-rbac + # - validate-minimal-rbac-override + # - validate-multi-namespace + # - validate-kots-pull + # - validate-app-version-label + # - validate-helm-install-order + # - validate-no-redeploy-on-restart + # - validate-kubernetes-installer-preflight + # - validate-kots-upgrade + # - validate-remove-app + # - validate-registry-check + # - validate-kots-helm-release-secret-migration + # - validate-native-helm-v2 + # - validate-deployment-orchestration + # - validate-replicated-sdk + # - validate-strict-preflight-checks + - validate-support-bundle + # # cli-only tests + # - validate-kots-push-images-anonymous steps: - run: echo "All PR tests passed"