From 71da2e8eabe5aa0e2749475eb864552483c37329 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 19 Jun 2024 10:38:05 +0200 Subject: [PATCH 1/7] Fix and enable instance details page UI tests --- .github/workflows/ui-test.yaml | 54 +++++++++++++++++++ .../botkube_page_helpers_test.go | 12 +++-- .../cloud_slack_dev_e2e_test.go | 11 ++-- 3 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/ui-test.yaml diff --git a/.github/workflows/ui-test.yaml b/.github/workflows/ui-test.yaml new file mode 100644 index 000000000..b485249d3 --- /dev/null +++ b/.github/workflows/ui-test.yaml @@ -0,0 +1,54 @@ +name: Test UI E2E tests on PR + +on: + push: + branches: + - enable-update-instance-ui + +env: + GIT_USER: botkube-dev + HELM_VERSION: v3.9.0 + K3D_VERSION: v5.4.6 + IMAGE_REGISTRY: "ghcr.io" + IMAGE_REPOSITORY: "kubeshop/botkube" + IMAGE_TAG: v9.99.9-dev # TODO: Use commit hash tag to make the predictable builds for each commit on branch + +jobs: + cloud-slack-dev-e2e: + name: Botkube Cloud Slack Dev E2E + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + concurrency: + group: cloud-slack-dev-e2e + cancel-in-progress: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run e2e tests + uses: ./.github/actions/cloud-slack-e2e + with: + access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} + + slack_workspace_name: ${{ secrets.E2E_DEV_SLACK_WORKSPACE_NAME }} + slack_email: ${{ secrets.E2E_DEV_SLACK_EMAIL }} + slack_password: ${{ secrets.E2E_DEV_SLACK_USER_PASSWORD }} + slack_bot_display_name: "BotkubeDev" + slack_tester_bot_token: ${{ secrets.E2E_DEV_SLACK_TESTER_BOT_TOKEN }} + slack_tester_bot_name: "botkubedev" + + botkube_cloud_api_base_url: "https://api-dev.botkube.io" + botkube_cloud_email: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_EMAIL }} + botkube_cloud_password: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_PASSWORD }} + botkube_cloud_team_organization_id: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID }} + + slack_alerts_webhook: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} + + e2e_type: "DEV" + + + + + diff --git a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go index f427553d9..f5534e803 100644 --- a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go @@ -203,14 +203,18 @@ func (p *BotkubeCloudPage) FinishWizard(t *testing.T) { func (p *BotkubeCloudPage) UpdateKubectlNamespace(t *testing.T) { t.Log("Updating 'kubectl' namespace property") - + p.openKubectlUpdateForm() - + p.page.MustElementR("input#root_defaultNamespace", "default").MustSelectAllText().MustInput("kube-system") p.page.Screenshot("after-changing-namespace-property") p.page.MustElementR("button", "/^Update$/i").MustClick() p.page.Screenshot("after-clicking-plugin-update") + t.Log("Moving to top left corner of the page") + p.page.Mouse.MustMoveTo(0, 0) + p.page.Screenshot("after-moving-to-top-left") + t.Log("Submitting changes") p.page.MustWaitStable() p.page.MustElementR("button", "/Deploy changes/i").MustClick() @@ -230,10 +234,10 @@ func (p *BotkubeCloudPage) openKubectlUpdateForm() { p.page.MustWaitStable() p.page.Screenshot("after-selecting-plugins-tab") - + p.page.MustElement(`button[id^="botkube/kubectl_"]`).MustClick() p.page.Screenshot("after-opening-kubectl-cfg") - + p.page.MustElement(`div[data-node-key="ui-form"]`).MustClick() p.page.Screenshot("after-selecting-kubectl-cfg-form") } diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index d86c71843..d5bd8b572 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -159,13 +159,10 @@ func TestCloudSlackE2E(t *testing.T) { botkubeCloudPage.FinishWizard(t) botkubeCloudPage.VerifyDeploymentStatus(t, "Connected") - if !isHeadless { // it is flaky on CI, more investigation needed - botkubeCloudPage.UpdateKubectlNamespace(t) - botkubeCloudPage.VerifyDeploymentStatus(t, "Updating") - botkubeCloudPage.VerifyDeploymentStatus(t, "Connected") - botkubeCloudPage.VerifyUpdatedKubectlNamespace(t) - } - + botkubeCloudPage.UpdateKubectlNamespace(t) + botkubeCloudPage.VerifyDeploymentStatus(t, "Updating") + botkubeCloudPage.VerifyDeploymentStatus(t, "Connected") + botkubeCloudPage.VerifyUpdatedKubectlNamespace(t) }) t.Run("Run E2E tests with deployment", func(t *testing.T) { From c4c170db464e258008208f48341ef4c07d3a9883 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 19 Jun 2024 11:21:09 +0200 Subject: [PATCH 2/7] Workaround glitch with "Outdated version detected" --- test/cloud-slack-dev-e2e/botkube_page_helpers_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go index f5534e803..e7871ee7b 100644 --- a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go @@ -235,7 +235,9 @@ func (p *BotkubeCloudPage) openKubectlUpdateForm() { p.page.MustWaitStable() p.page.Screenshot("after-selecting-plugins-tab") - p.page.MustElement(`button[id^="botkube/kubectl_"]`).MustClick() + p.page.MustElement(`button[id^="botkube/kubectl_"]`). + MustWaitEnabled(). // needed as we have an "Outdated version detected" glitch + MustClick() p.page.Screenshot("after-opening-kubectl-cfg") p.page.MustElement(`div[data-node-key="ui-form"]`).MustClick() From f56b9fe5972a03b279fae0d5c1425262decdf206 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 19 Jun 2024 11:57:06 +0200 Subject: [PATCH 3/7] Add sleep for wizard buttons --- test/cloud-slack-dev-e2e/botkube_page_helpers_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go index e7871ee7b..cf367f704 100644 --- a/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/botkube_page_helpers_test.go @@ -174,6 +174,7 @@ func (p *BotkubeCloudPage) FinishWizard(t *testing.T) { t.Log("Navigating to plugin selection") p.page.Screenshot("before-first-next") + time.Sleep(3 * time.Second) p.page.MustElementR("button", "/^Next$/i"). MustWaitEnabled(). // We need to wait, otherwise, we click the same 'Next' button twice before the query is executed, and we are not really @@ -183,6 +184,7 @@ func (p *BotkubeCloudPage) FinishWizard(t *testing.T) { p.page.Screenshot("after-first-next") t.Log("Using pre-selected plugins. Navigating to wizard summary") + time.Sleep(3 * time.Second) p.page.MustElementR("button", "/^Next$/i"). MustWaitEnabled(). // We need to wait, otherwise, we click the same 'Next' button twice before the query is executed, and we are not really @@ -191,12 +193,13 @@ func (p *BotkubeCloudPage) FinishWizard(t *testing.T) { p.page.Screenshot("after-second-next") t.Log("Submitting changes") + time.Sleep(3 * time.Second) p.page.MustElementR("button", "/^Deploy changes$/i"). MustWaitEnabled(). MustClick() p.page.Screenshot("after-deploy-changes") - // wait till gql mutation passes, and navigates to install details, otherwise, we could navigate to instance details with state 'draft' + // wait till gql mutation passes, and navigates to instance details, otherwise, we could navigate to instance details with state 'draft' p.page.MustWaitNavigation() p.page.Screenshot("after-deploy-changes-navigation") } From 596d6da9294b176b8d975325b8edc439de9793a1 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 19 Jun 2024 12:55:14 +0200 Subject: [PATCH 4/7] Improve resiliency --- .../slack_page_helpers_test.go | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go index e516c2c93..26c67ef32 100644 --- a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go @@ -3,13 +3,17 @@ package cloud_slack_dev_e2e import ( + "github.com/stretchr/testify/assert" "testing" "time" "github.com/go-rod/rod" ) -const slackBaseURL = "slack.com" +const ( + slackBaseURL = "slack.com" + waitTime = 10 * time.Second +) type SlackPage struct { page *Page @@ -63,11 +67,33 @@ func (p *SlackPage) ConnectWorkspace(t *testing.T, headless bool, browser *rod.B p.page.MustElementR("div.ant-result-title", "Organization Already Connected!") } else { t.Log("Finalizing connection...") + time.Sleep(3 * time.Second) p.page.Screenshot() - p.page.MustElement("button#slack-workspace-connect").MustClick(). - MustWaitEnabled() // when it's re-enabled, then it means the query was finished + p.page.MustElement("button#slack-workspace-connect").MustClick() p.page.Screenshot() } + p.waitForHomepage(t) + _ = p.page.Close() // the page should be closed automatically anyway } + +func (p *SlackPage) waitForHomepage(t *testing.T) { + t.Log("Detecting homepage...") + time.Sleep(waitTime) // ensure the screenshots shows a view after button click + p.page.Screenshot() + + // Case 1: There are other instances on the list + shortBkTimeoutPage := p.page.Timeout(waitTime) + t.Cleanup(func() { + closePage(t, "shortBkTimeoutPage", shortBkTimeoutPage) + }) + _, err := shortBkTimeoutPage.ElementR(".ant-layout-content p", "All Botkube installations managed by Botkube Cloud.") + if err != nil { + t.Logf("Failed to detect homepage with other instances created: %v", err) + // Fallback to Case 2: No other instances created + t.Logf("Checking if the homepage is in the 'no instances' state...") + _, err := p.page.ElementR(".ant-layout-content h2", "Create your Botkube instance!") + assert.NoError(t, err) + } +} From 84fb8fab18f99c1b46cc0dd54b11dbe008e48103 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 20 Jun 2024 00:18:04 +0200 Subject: [PATCH 5/7] Fix assertion for connected Slack --- .../cloud_slack_dev_e2e_test.go | 4 +- .../slack_page_helpers_test.go | 89 +++++++++---------- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go index d5bd8b572..3ae3b18d2 100644 --- a/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go +++ b/test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go @@ -42,7 +42,7 @@ type E2ESlackConfig struct { Slack SlackConfig BotkubeCloud BotkubeCloudConfig - PageTimeout time.Duration `envconfig:"default=5m"` + PageTimeout time.Duration `envconfig:"default=10m"` ScreenshotsDir string `envconfig:"optional"` DebugMode bool `envconfig:"default=false"` @@ -152,7 +152,7 @@ func TestCloudSlackE2E(t *testing.T) { botkubeCloudPage.InstallAgentInCluster(t, cfg.BotkubeCliBinaryPath) botkubeCloudPage.OpenSlackAppIntegrationPage(t) - slackPage.ConnectWorkspace(t, isHeadless, browser) + slackPage.ConnectWorkspace(t, browser) botkubeCloudPage.ReAddSlackPlatformIfShould(t, isHeadless) botkubeCloudPage.SetupSlackWorkspace(t, channel.Name()) diff --git a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go index 26c67ef32..0642dde4b 100644 --- a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go @@ -3,6 +3,8 @@ package cloud_slack_dev_e2e import ( + "context" + "errors" "github.com/stretchr/testify/assert" "testing" "time" @@ -11,89 +13,82 @@ import ( ) const ( - slackBaseURL = "slack.com" - waitTime = 10 * time.Second + slackBaseURL = "slack.com" + waitTime = 10 * time.Second + contextTimeout = 30 * time.Second + shorterContextTimeout = 10 * time.Second ) type SlackPage struct { page *Page - cfg SlackConfig + cfg E2ESlackConfig } func NewSlackPage(t *testing.T, cfg E2ESlackConfig) *SlackPage { return &SlackPage{ page: &Page{t: t, cfg: cfg}, - cfg: cfg.Slack, + cfg: cfg, } } -func (p *SlackPage) ConnectWorkspace(t *testing.T, headless bool, browser *rod.Browser) { +func (p *SlackPage) ConnectWorkspace(t *testing.T, browser *rod.Browser) { p.page.Page = browser.MustPages().MustFindByURL(slackBaseURL) + p.page.MustWaitStable() - p.page.MustElement("input#domain").MustInput(p.cfg.WorkspaceName) - + defer func(page *Page) { + err := page.Close() + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + t.Fatalf("Failed to close page: %s", err.Error()) + } + }(p.page) + + p.page.MustElement("input#domain").MustInput(p.cfg.Slack.WorkspaceName) p.page.MustElementR("button", "Continue").MustClick() - p.page.Screenshot() - - // here we get reloaded, so we need to type it again (looks like bug on Slack side) - if !headless { - p.page.MustElement("input#domain").MustInput(p.cfg.WorkspaceName) - p.page.MustElementR("button", "Continue").MustClick() - } + p.page.Screenshot("after-continue") p.page.MustWaitStable() p.page.MustElementR("a", "sign in with a password instead").MustClick() - p.page.Screenshot() - p.page.MustElement("input#email").MustInput(p.cfg.Email) - p.page.MustElement("input#password").MustInput(p.cfg.Password) + p.page.Screenshot("after-sign-in-with-password") + p.page.MustElement("input#email").MustInput(p.cfg.Slack.Email) + p.page.MustElement("input#password").MustInput(p.cfg.Slack.Password) p.page.Screenshot() t.Log("Hide Slack cookie banner that collides with 'Sign in' button") - cookie, err := p.page.Timeout(5 * time.Second).Element("button#onetrust-accept-btn-handler") + pageWithTimeout := p.page.Timeout(shorterContextTimeout) + t.Cleanup(func() { + _ = pageWithTimeout.Close() + }) + + cookieElem, err := pageWithTimeout.Element("button#onetrust-accept-btn-handler") if err != nil { t.Logf("Failed to obtain cookie element: %s. Skipping...", err.Error()) } else { - cookie.MustClick() + cookieElem.MustClick() } p.page.MustElementR("button", "/^Sign in$/i").MustClick() - p.page.Screenshot() + p.page.Screenshot("after-sign-in") + time.Sleep(waitTime) // ensure the screenshots shows a page after "Sign in" click + p.page.Screenshot("after-sign-in-page") p.page.MustElementR("button.c-button:not(.c-button--disabled)", "Allow").MustClick() t.Log("Finalizing Slack workspace connection...") - if p.cfg.WorkspaceAlreadyConnected { + if p.cfg.Slack.WorkspaceAlreadyConnected { t.Log("Expecting already connected message...") p.page.MustElementR("div.ant-result-title", "Organization Already Connected!") } else { t.Log("Finalizing connection...") - time.Sleep(3 * time.Second) - p.page.Screenshot() + time.Sleep(waitTime) + p.page.Screenshot("before-workspace-connect") p.page.MustElement("button#slack-workspace-connect").MustClick() - p.page.Screenshot() + p.page.Screenshot("after-workspace-connect") } - p.waitForHomepage(t) - - _ = p.page.Close() // the page should be closed automatically anyway -} - -func (p *SlackPage) waitForHomepage(t *testing.T) { - t.Log("Detecting homepage...") - time.Sleep(waitTime) // ensure the screenshots shows a view after button click - p.page.Screenshot() - - // Case 1: There are other instances on the list - shortBkTimeoutPage := p.page.Timeout(waitTime) - t.Cleanup(func() { - closePage(t, "shortBkTimeoutPage", shortBkTimeoutPage) - }) - _, err := shortBkTimeoutPage.ElementR(".ant-layout-content p", "All Botkube installations managed by Botkube Cloud.") - if err != nil { - t.Logf("Failed to detect homepage with other instances created: %v", err) - // Fallback to Case 2: No other instances created - t.Logf("Checking if the homepage is in the 'no instances' state...") - _, err := p.page.ElementR(".ant-layout-content h2", "Create your Botkube instance!") - assert.NoError(t, err) - } + _, err = p.page.Element("#non-existing-elem") + // expected context canceled = which means, it was auto-closed + assert.EqualError(t, err, context.Canceled.Error()) } From 6f9495d9c64e9feda595ee55bcb636520e29921a Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 20 Jun 2024 00:24:12 +0200 Subject: [PATCH 6/7] Improve waiting for page auto-close --- test/cloud-slack-dev-e2e/slack_page_helpers_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go index 0642dde4b..f6397885d 100644 --- a/test/cloud-slack-dev-e2e/slack_page_helpers_test.go +++ b/test/cloud-slack-dev-e2e/slack_page_helpers_test.go @@ -88,7 +88,7 @@ func (p *SlackPage) ConnectWorkspace(t *testing.T, browser *rod.Browser) { p.page.Screenshot("after-workspace-connect") } - _, err = p.page.Element("#non-existing-elem") - // expected context canceled = which means, it was auto-closed - assert.EqualError(t, err, context.Canceled.Error()) + t.Log("Waiting for page auto-close...") + err = p.page.WaitIdle(waitTime) // wait for auto-close + assert.NoError(t, err) } From 160dd3f35c56087759492d830204ea07a4362cd1 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 20 Jun 2024 09:23:50 +0200 Subject: [PATCH 7/7] Remove test job --- .github/workflows/ui-test.yaml | 54 ---------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 .github/workflows/ui-test.yaml diff --git a/.github/workflows/ui-test.yaml b/.github/workflows/ui-test.yaml deleted file mode 100644 index b485249d3..000000000 --- a/.github/workflows/ui-test.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: Test UI E2E tests on PR - -on: - push: - branches: - - enable-update-instance-ui - -env: - GIT_USER: botkube-dev - HELM_VERSION: v3.9.0 - K3D_VERSION: v5.4.6 - IMAGE_REGISTRY: "ghcr.io" - IMAGE_REPOSITORY: "kubeshop/botkube" - IMAGE_TAG: v9.99.9-dev # TODO: Use commit hash tag to make the predictable builds for each commit on branch - -jobs: - cloud-slack-dev-e2e: - name: Botkube Cloud Slack Dev E2E - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - concurrency: - group: cloud-slack-dev-e2e - cancel-in-progress: false - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run e2e tests - uses: ./.github/actions/cloud-slack-e2e - with: - access_token: ${{ secrets.E2E_TEST_GH_DEV_ACCOUNT_PAT }} - - slack_workspace_name: ${{ secrets.E2E_DEV_SLACK_WORKSPACE_NAME }} - slack_email: ${{ secrets.E2E_DEV_SLACK_EMAIL }} - slack_password: ${{ secrets.E2E_DEV_SLACK_USER_PASSWORD }} - slack_bot_display_name: "BotkubeDev" - slack_tester_bot_token: ${{ secrets.E2E_DEV_SLACK_TESTER_BOT_TOKEN }} - slack_tester_bot_name: "botkubedev" - - botkube_cloud_api_base_url: "https://api-dev.botkube.io" - botkube_cloud_email: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_EMAIL }} - botkube_cloud_password: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_PASSWORD }} - botkube_cloud_team_organization_id: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID }} - - slack_alerts_webhook: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }} - - e2e_type: "DEV" - - - - -