diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 1169ab70f0..e995132d48 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -697,37 +697,110 @@ jobs: strategy: fail-fast: false matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + cluster: [ + {distribution: kind, version: v1.28.0}, + {distribution: openshift, version: 4.13.0-okd} + ] + env: + APP_SLUG: minimal-rbac steps: - name: Checkout uses: actions/checkout@v3 - - name: download e2e deps - uses: actions/download-artifact@v3 + + - name: Create Cluster + id: create-cluster + uses: replicatedhq/replicated-actions/create-cluster@v1 with: - name: e2e - path: e2e/bin/ - - run: docker load -i e2e/bin/e2e-deps.tar - - run: chmod +x e2e/bin/* + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + kubernetes-distribution: ${{ matrix.cluster.distribution }} + kubernetes-version: ${{ matrix.cluster.version }} + cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }} + timeout-minutes: '120' + ttl: 2h + export-kubeconfig: true + - name: download kots binary uses: actions/download-artifact@v3 with: name: kots path: bin/ - - run: chmod +x bin/* - - uses: ./.github/actions/kots-e2e + + - run: chmod +x bin/kots + + - name: create namespace and dockerhub secret + run: | + kubectl create ns "$APP_SLUG" + kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG" + + - name: run the test + run: | + set +e + echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml + ./bin/kots \ + install "$APP_SLUG/automated" \ + --license-file license.yaml \ + --no-port-forward \ + --namespace "$APP_SLUG" \ + --shared-password password \ + --kotsadm-registry ttl.sh \ + --kotsadm-namespace automated-${{ github.run_id }} \ + --kotsadm-tag 24h | tee output.txt + + if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + echo "Expected to see an RBAC error for preflight checks, but did not" + exit 1 + fi + + if ! grep -q 'The app was not deployed.' output.txt; then + printf "Expected to see message about app not being deployed, but did not\n" + exit 1 + fi + + if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + printf "Did not expect to see a failure about number of nodes, but did\n" + exit 1 + fi + + if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm role not found in namespace $APP_SLUG" + kubectl get role -n "$APP_SLUG" + exit 1 + fi + + if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm rolebinding not found in namespace $APP_SLUG" + kubectl get rolebinding -n "$APP_SLUG" + exit 1 + fi + + if kubectl get clusterrole | grep -q kotsadm; then + echo "found kotsadm clusterrole in minimal RBAC install" + kubectl get clusterrole + exit 1 + fi + + if kubectl get clusterrolebinding | grep -q kotsadm; then + echo "found kotsadm clusterrolebinding in minimal RBAC install" + kubectl get clusterrolebinding + exit 1 + fi + + - name: Generate support bundle on failure + if: failure() + uses: ./.github/actions/generate-support-bundle with: - test-focus: 'Minimal RBAC' - kots-namespace: 'minimal-rbac' - k8s-distribution: ${{ matrix.cluster.distribution }} - k8s-version: ${{ matrix.cluster.version }} - testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}' - testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }} + kots-namespace: "$APP_SLUG" aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}' aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}' - replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}' - kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}' - kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}' + + - name: Remove Cluster + id: remove-cluster + uses: replicatedhq/replicated-actions/remove-cluster@v1 + if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }} + continue-on-error: true + with: + api-token: ${{ secrets.C11Y_MATRIX_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} validate-backup-and-restore: @@ -776,7 +849,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -816,7 +889,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: strict-preflight-checks @@ -925,7 +998,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -1004,7 +1077,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -1043,8 +1116,10 @@ jobs: strategy: fail-fast: false matrix: - cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }} - continue-on-error: ${{ matrix.cluster.stage != 'stable' }} + cluster: [ + {distribution: kind, version: v1.28.0}, + {distribution: openshift, version: 4.13.0-okd} + ] env: APP_SLUG: minimal-rbac APP_VERSION_LABEL: "0.0.1" @@ -1080,8 +1155,11 @@ jobs: - name: minimal rbac override on command line run: | + set +e + echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml ./bin/kots \ install "$APP_SLUG/automated" \ + --license-file license.yaml \ --app-version-label "$APP_VERSION_LABEL" \ --no-port-forward \ --namespace "$APP_SLUG" \ @@ -1089,16 +1167,45 @@ jobs: --kotsadm-registry ttl.sh \ --kotsadm-namespace automated-${{ github.run_id }} \ --kotsadm-tag 24h \ - --use-minimal-rbac - if kubectl get roles -n "$APP_SLUG" | grep -q kotsadm; then - echo "Found kotsadm role in ${APP_SLUG}" - else - echo "No kotsadm role found in appication namespace" + --use-minimal-rbac | tee output.txt + + if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + echo "Expected to see an RBAC error for preflight checks, but did not" exit 1 fi - if kubectl get clusterroles | grep -q kotsadm; then - echo "Found kotsadm cluster roles in minimal RBAC install" - exit + + if ! grep -q 'The app was not deployed.' output.txt; then + printf "Expected to see message about app not being deployed, but did not\n" + exit 1 + fi + + if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + printf "Did not expect to see a failure about number of nodes, but did\n" + exit 1 + fi + + if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm role not found in namespace $APP_SLUG" + kubectl get role -n "$APP_SLUG" + exit 1 + fi + + if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm rolebinding not found in namespace $APP_SLUG" + kubectl get rolebinding -n "$APP_SLUG" + exit 1 + fi + + if kubectl get clusterrole | grep -q kotsadm; then + echo "found kotsadm clusterrole in minimal RBAC install" + kubectl get clusterrole + exit 1 + fi + + if kubectl get clusterrolebinding | grep -q kotsadm; then + echo "found kotsadm clusterrolebinding in minimal RBAC install" + kubectl get clusterrolebinding + exit 1 fi - name: create namespace and dockerhub secret @@ -1109,25 +1216,58 @@ jobs: - name: no minimal rbac override on command line run: | + set +e + echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml ./bin/kots \ install "$APP_SLUG/automated" \ + --license-file license.yaml \ --app-version-label "$APP_VERSION_LABEL" \ --no-port-forward \ --namespace "$APP_SLUG" \ --shared-password password \ --kotsadm-registry ttl.sh \ --kotsadm-namespace automated-${{ github.run_id }} \ - --kotsadm-tag 24h - if kubectl get roles -n "$APP_SLUG" | grep -q kotsadm; then - echo "Found kotsadm role in cluster scoped install" + --kotsadm-tag 24h | tee output.txt + + if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then + echo "Did not expect to see an RBAC error for preflight checks, but did" exit 1 fi - if kubectl get clusterroles | grep -q kotsadm; then - echo "Found kotsadm cluster role in cluster scoped install" - else - echo "No kotsadm cluster role in cluster scoped install" + + if ! grep -q 'The app was not deployed.' output.txt; then + printf "Expected to see message about app not being deployed, but did not\n" + exit 1 + fi + + if ! grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then + printf "Expected to see a failure about number of nodes, but did not\n" + exit 1 + fi + + if kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm role found in cluster scoped install" + kubectl get role -n "$APP_SLUG" + exit 1 + fi + + if kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then + echo "kotsadm rolebinding found in cluster scoped install" + kubectl get rolebinding -n "$APP_SLUG" + exit 1 + fi + + if ! kubectl get clusterrole | grep -q kotsadm; then + echo "No kotsadm clusterrole found in cluster scoped install" + kubectl get clusterrole + exit 1 + fi + + if ! kubectl get clusterrolebinding | grep -q kotsadm; then + echo "No kotsadm clusterrolebinding found in cluster scoped install" + kubectl get clusterrolebinding exit 1 fi + - name: Generate support bundle on failure if: failure() uses: ./.github/actions/generate-support-bundle @@ -1375,7 +1515,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: app-version-label @@ -1632,7 +1772,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: no-redeploy-on-restart @@ -1754,7 +1894,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: kubernetes-installer-preflight @@ -1891,7 +2031,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -1952,7 +2092,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: minimal-rbac @@ -2149,7 +2289,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2217,7 +2357,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2284,7 +2424,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2490,7 +2630,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: helm-release-secret-migration @@ -2677,7 +2817,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2717,7 +2857,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2757,7 +2897,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -2798,7 +2938,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27, instance-type: r1.medium} + {distribution: kind, version: v1.28.0, instance-type: r1.medium} ] env: APP_SLUG: remove-app @@ -2977,7 +3117,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] steps: - name: Checkout @@ -3072,7 +3212,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: native-helm-v2 @@ -3298,7 +3438,7 @@ jobs: fail-fast: false matrix: cluster: [ - {distribution: kind, version: v1.27.0} + {distribution: kind, version: v1.28.0} ] env: APP_SLUG: deployment-orchestration @@ -3468,7 +3608,7 @@ jobs: cluster: [ {distribution: kind, version: v1.28.0}, {distribution: openshift, version: 4.13.0-okd} - ] + ] env: KOTS_NAMESPACE: replicated-sdk steps: @@ -3780,7 +3920,6 @@ jobs: # testim tests - validate-existing-online-install-minimal - validate-smoke-test - - validate-minimal-rbac - validate-backup-and-restore - validate-no-required-config - validate-version-history-pagination @@ -3794,6 +3933,7 @@ jobs: - validate-airgap-smoke-test - validate-config # non-testim tests + - validate-minimal-rbac - validate-minimal-rbac-override - validate-multi-namespace - validate-kots-pull diff --git a/cmd/kots/cli/install.go b/cmd/kots/cli/install.go index 84170c7dbc..7ca212a246 100644 --- a/cmd/kots/cli/install.go +++ b/cmd/kots/cli/install.go @@ -32,6 +32,7 @@ import ( "github.com/replicatedhq/kots/pkg/kurl" "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/metrics" + preflighttypes "github.com/replicatedhq/kots/pkg/preflight/types" "github.com/replicatedhq/kots/pkg/print" "github.com/replicatedhq/kots/pkg/pull" "github.com/replicatedhq/kots/pkg/replicatedapp" @@ -447,14 +448,14 @@ func InstallCmd() *cobra.Command { log.FinishSpinner() switch status { - case storetypes.VersionPendingPreflight: + case storetypes.VersionPendingPreflight, storetypes.VersionPending: log.ActionWithSpinner("Waiting for preflight checks to complete") if err := ValidatePreflightStatus(deployOptions, authSlug, apiEndpoint); err != nil { perr := preflightError{} if errors.As(err, &perr) { log.FinishSpinner() // We succeeded waiting for the results. Don't finish with an error log.Errorf(perr.Msg) - print.PreflightErrors(log, perr.Results) + print.PreflightResults(perr.Results) cmd.SilenceErrors = true // Stop Cobra from printing the error, we format the message ourselves } else { log.FinishSpinnerWithError() @@ -1020,7 +1021,7 @@ func checkPreflightsComplete(response *handlers.GetPreflightResultResponse) (boo type preflightError struct { Msg string - Results []*preflight.UploadPreflightResult + Results preflighttypes.PreflightResults } func (e preflightError) Error() string { @@ -1034,12 +1035,30 @@ func checkPreflightResults(response *handlers.GetPreflightResultResponse) (bool, return false, nil } - var results preflight.UploadPreflightResults + var results preflighttypes.PreflightResults err := json.Unmarshal([]byte(response.PreflightResult.Result), &results) if err != nil { return false, errors.Wrap(err, fmt.Sprintf("failed to unmarshal upload preflight results from response: %v", response.PreflightResult.Result)) } + if len(results.Errors) > 0 { + isRBAC := false + for _, err := range results.Errors { + if err.IsRBAC { + isRBAC = true + break + } + } + msg := "There are preflight check errors for the application. The app was not deployed." + if isRBAC { + msg = "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding. The app was not deployed." + } + return false, preflightError{ + Msg: msg, + Results: results, + } + } + var isWarn, isFail bool for _, result := range results.Results { if result.IsWarn { @@ -1053,20 +1072,20 @@ func checkPreflightResults(response *handlers.GetPreflightResultResponse) (bool, if isWarn && isFail { return false, preflightError{ Msg: "There are preflight check failures and warnings for the application. The app was not deployed.", - Results: results.Results, + Results: results, } } if isWarn { return false, preflightError{ Msg: "There are preflight check warnings for the application. The app was not deployed.", - Results: results.Results, + Results: results, } } if isFail { return false, preflightError{ Msg: "There are preflight check failures for the application. The app was not deployed.", - Results: results.Results, + Results: results, } } diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 9eb4877142..230fe47646 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -210,7 +210,6 @@ var _ = Describe("E2E", func() { Entry(nil, inventory.NewSmokeTest()), Entry(nil, inventory.NewAirgapSmokeTest()), Entry(nil, inventory.NewConfigValidation()), - Entry(nil, inventory.NewMinimalRBACTest()), Entry(nil, inventory.NewBackupAndRestore()), Entry(nil, inventory.NewNoRequiredConfig()), Entry(nil, inventory.NewVersionHistoryPagination()), diff --git a/e2e/testim/inventory/inventory.go b/e2e/testim/inventory/inventory.go index d5f3ce917f..70c7eb39ef 100644 --- a/e2e/testim/inventory/inventory.go +++ b/e2e/testim/inventory/inventory.go @@ -65,15 +65,6 @@ func NewConfigValidation() Test { } } -func NewMinimalRBACTest() Test { - return Test{ - Name: "Minimal RBAC", - Suite: "minimal-rbac", - Namespace: "minimal-rbac", - UpstreamURI: "minimal-rbac/automated", - } -} - func NewBackupAndRestore() Test { return Test{ Name: "Backup and Restore", diff --git a/hack/build-ttl.sh b/hack/build-ttl.sh index a6a625be17..7d2aafe493 100755 --- a/hack/build-ttl.sh +++ b/hack/build-ttl.sh @@ -12,4 +12,4 @@ printf "\n\n\n" printf "Run command: ${GREEN}kubectl edit deployment kotsadm${NC}\n" printf "Replace image with: ${GREEN}ttl.sh/${CURRENT_USER}/kotsadm:24h${NC}\n" printf "\n" -printf "These images are good for 12 hours\n" +printf "These images are good for 24 hours\n" diff --git a/migrations/build-ttl.sh b/migrations/build-ttl.sh index ed3e921fe1..32a9769e95 100755 --- a/migrations/build-ttl.sh +++ b/migrations/build-ttl.sh @@ -14,4 +14,4 @@ printf "\n\n\n" printf "Run command: ${GREEN}kubectl edit deployment kotsadm${NC}\n" printf "Replace image with: ${GREEN}${IMAGE}${NC}\n" printf "\n" -printf "This image is good for 12 hours\n" +printf "This image is good for 24 hours\n" diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go index edcfea6e4e..4ec02d270d 100644 --- a/pkg/preflight/preflight.go +++ b/pkg/preflight/preflight.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/pkg/errors" apptypes "github.com/replicatedhq/kots/pkg/app/types" @@ -277,6 +278,9 @@ func maybeDeployFirstVersion(appID string, sequence int64, preflightResults *typ return false, errors.Wrap(err, "failed to deploy version") } + // this ensures the version status is updated before returning + time.Sleep(time.Second) + return true, nil } diff --git a/pkg/print/config.go b/pkg/print/config.go index 3507dcc75a..adadbbe4cf 100644 --- a/pkg/print/config.go +++ b/pkg/print/config.go @@ -6,8 +6,6 @@ import ( configtypes "github.com/replicatedhq/kots/pkg/kotsadmconfig/types" "github.com/replicatedhq/kots/pkg/logger" - "github.com/replicatedhq/kots/pkg/preflight" - tsPreflight "github.com/replicatedhq/troubleshoot/pkg/preflight" ) func ConfigValidationErrors(log *logger.CLILogger, groupValidationErrors []configtypes.ConfigGroupValidationError) { @@ -28,16 +26,3 @@ func ConfigValidationErrors(log *logger.CLILogger, groupValidationErrors []confi log.FinishSpinnerWithError() log.Errorf(sb.String()) } - -func PreflightErrors(log *logger.CLILogger, results []*tsPreflight.UploadPreflightResult) { - w := NewTabWriter() - defer w.Flush() - - fmt.Fprintf(w, "\n") - fmtColumns := "%s\t%s\t%s\n" - fmt.Fprintf(w, fmtColumns, "STATE", "TITLE", "MESSAGE") - for _, result := range results { - fmt.Fprintf(w, fmtColumns, strings.ToUpper(preflight.GetPreflightCheckState(result)), result.Title, result.Message) - } - fmt.Fprintf(w, "\n") -} diff --git a/pkg/print/preflight.go b/pkg/print/preflight.go new file mode 100644 index 0000000000..5020363943 --- /dev/null +++ b/pkg/print/preflight.go @@ -0,0 +1,33 @@ +package print + +import ( + "fmt" + "strings" + + "github.com/replicatedhq/kots/pkg/preflight" + preflighttypes "github.com/replicatedhq/kots/pkg/preflight/types" +) + +func PreflightResults(results preflighttypes.PreflightResults) { + w := NewTabWriter() + defer w.Flush() + + if len(results.Errors) > 0 { + fmt.Fprintf(w, "\n") + for _, err := range results.Errors { + fmtColumns := " - %s\n" + fmt.Fprintf(w, fmtColumns, err.Error) + } + fmt.Fprintf(w, "\n") + } + + if len(results.Results) > 0 { + fmt.Fprintf(w, "\n") + fmtColumns := "%s\t%s\t%s\n" + fmt.Fprintf(w, fmtColumns, "STATE", "TITLE", "MESSAGE") + for _, result := range results.Results { + fmt.Fprintf(w, fmtColumns, strings.ToUpper(preflight.GetPreflightCheckState(result)), result.Title, result.Message) + } + fmt.Fprintf(w, "\n") + } +}