From 256ee81730084ef2bb1d7bf016cbb5cbc0870c5d Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:00:29 +0000 Subject: [PATCH 1/7] Attach / upload Playwright report on failure --- .github/actions/kots-e2e/action.yml | 6 ++++++ e2e/Dockerfile | 2 +- e2e/Makefile | 2 ++ e2e/playwright/playwright.config.ts | 5 ++++- e2e/playwright/tests/smoke-test/test.spec.ts | 4 ++-- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index b3ad4833ff..91caba23c8 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -123,6 +123,12 @@ runs: SKIP_TEARDOWN=1 shell: bash + - uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: ${{ inputs.test-focus }} Playwright report + path: ./e2e/playwright/playwright-report/ + - name: Print logs on failure if: ${{ failure() }} env: diff --git a/e2e/Dockerfile b/e2e/Dockerfile index cc0007b047..693082d493 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/e2e/Makefile b/e2e/Makefile index 94f9bc1d1b..f97dae0c1a 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -2,6 +2,7 @@ include ../Makefile.build.mk BIN_DIR := $(shell pwd)/bin KOTS_BIN_DIR := $(shell dirname $(shell pwd))/bin +PLAYWRIGHT_DIR := $(shell pwd)/playwright SHELL := /bin/bash @@ -32,6 +33,7 @@ test: -v $(BIN_DIR)/e2e.test:/usr/local/bin/e2e.test \ -v $(KOTS_BIN_DIR)/kots:/usr/local/bin/kots \ -v $(KOTS_BIN_DIR)/kots:/usr/local/bin/kubectl-kots \ + -v $(PLAYWRIGHT_DIR)/playwright-report:/playwright/playwright-report \ $(EXISTING_KUBECONFIG_VOLUME_MOUNT) \ -v /var/run/docker.sock:/var/run/docker.sock \ e2e-deps \ diff --git a/e2e/playwright/playwright.config.ts b/e2e/playwright/playwright.config.ts index fa279694a2..1ff9396059 100644 --- a/e2e/playwright/playwright.config.ts +++ b/e2e/playwright/playwright.config.ts @@ -20,7 +20,10 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'list', + reporter: [ + ['line'], + ['html', { open: process.env.CI ? 'never' : 'on-failure' }], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index a787121d36..5e50e08b1a 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -21,7 +21,7 @@ test('smoke test', async ({ page }) => { await page.getByRole('button', { name: 'Deploy' }).click(); await page.getByRole('button', { name: 'Deploy anyway' }).click(); await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 }); - await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 15000 }); + await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 30000 }); await expect(page.locator('#app')).toContainText('Check for update'); await expect(page.locator('#app')).toContainText('Configure automatic updates'); await expect(page.locator('#app')).toContainText('Redeploy', { timeout: 15000 }); @@ -190,6 +190,6 @@ test('smoke test', async ({ page }) => { await page.locator('div').filter({ hasText: /^Change passwordAdd new applicationLog out$/ }).getByRole('img').click(); await page.getByText('Log out', { exact: true }).click(); await expect(page.getByPlaceholder('password')).toBeVisible({ timeout: 30000 }); - await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin console.'); + await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin consolee.'); await expect(page.getByRole('button')).toContainText('Log in'); }); From 9c86748c1241d54d46809d695017f85c0272e109 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:16:28 +0000 Subject: [PATCH 2/7] unique artifact name --- .github/actions/kots-e2e/action.yml | 2 +- e2e/playwright/tests/smoke-test/test.spec.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index 91caba23c8..941448119b 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -126,7 +126,7 @@ runs: - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: - name: ${{ inputs.test-focus }} Playwright report + name: ${{ github.job }}-${{ inputs.k8s-distribution }}-${{ inputs.k8s-version }}-playwright-report path: ./e2e/playwright/playwright-report/ - name: Print logs on failure diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index 5e50e08b1a..1bc0e24fc6 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -7,7 +7,7 @@ test('smoke test', async ({ page }) => { test.setTimeout(5 * 60 * 1000); // 5 minutes await login(page); await uploadLicense(page, expect); - await expect(page.locator('#app')).toContainText('Install in airgapped environment', { timeout: 15000 }); + await expect(page.locator('#app')).toContainText('Install in airgapped environmentt', { timeout: 15000 }); await page.getByText('download App Name from the Internet').click(); await expect(page.locator('#app')).toContainText('Installing your license'); await expect(page.locator('h3')).toContainText('My Example Config', { timeout: 30000 }); @@ -190,6 +190,6 @@ test('smoke test', async ({ page }) => { await page.locator('div').filter({ hasText: /^Change passwordAdd new applicationLog out$/ }).getByRole('img').click(); await page.getByText('Log out', { exact: true }).click(); await expect(page.getByPlaceholder('password')).toBeVisible({ timeout: 30000 }); - await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin consolee.'); + await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin console.'); await expect(page.getByRole('button')).toContainText('Log in'); }); From b20e5557f0da1bb349023275c41a8c7be2b18d05 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:19:40 +0000 Subject: [PATCH 3/7] never open html report --- e2e/playwright/playwright.config.ts | 2 +- e2e/playwright/tests/smoke-test/test.spec.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/playwright/playwright.config.ts b/e2e/playwright/playwright.config.ts index 1ff9396059..edc37473bf 100644 --- a/e2e/playwright/playwright.config.ts +++ b/e2e/playwright/playwright.config.ts @@ -22,7 +22,7 @@ export default defineConfig({ /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ ['line'], - ['html', { open: process.env.CI ? 'never' : 'on-failure' }], + ['html', { open: 'never' }], ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index 1bc0e24fc6..6fe013fdcb 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -41,6 +41,7 @@ test('smoke test', async ({ page }) => { await expect(page.locator('.ConfigureUpdatesModal')).toContainText('At 12:00 AM, only on Sunday'); await page.getByRole('button', { name: 'Update', exact: true }).click(); await expect(page.getByText('Automatically check for updates', { exact: true })).not.toBeVisible(); + await page.waitForTimeout(2000); await page.locator('span[data-tip="View deploy logs"]').first().click(); await validateDeployLogs(page, expect); await page.reload(); From c7c94d9766f056c1a093d21d4a8dbd4d86188ab0 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:40:06 +0000 Subject: [PATCH 4/7] include screenshot on failure --- e2e/playwright/playwright.config.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/e2e/playwright/playwright.config.ts b/e2e/playwright/playwright.config.ts index edc37473bf..f9e8377bab 100644 --- a/e2e/playwright/playwright.config.ts +++ b/e2e/playwright/playwright.config.ts @@ -11,14 +11,6 @@ import { defineConfig, devices } from '@playwright/test'; */ export default defineConfig({ testDir: './tests', - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ ['line'], @@ -29,8 +21,15 @@ export default defineConfig({ /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: `http://localhost:${process.env.PORT || 8800}`, - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + /* + To include traces for failed tests, set this to 'retain-on-failure'. + This is not enabled by default because it's performance heavy. + See https://playwright.dev/docs/trace-viewer. + */ + trace: 'off', + + /* Screenshot on failure. */ + screenshot: 'only-on-failure', }, /* Configure projects for major browsers */ From f055d52f6c440f1d9f7b6dd2cc9e86cde5e90cb6 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:40:24 +0000 Subject: [PATCH 5/7] test with logs --- e2e/playwright/tests/smoke-test/test.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index 6fe013fdcb..f1a39ed62b 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -7,7 +7,7 @@ test('smoke test', async ({ page }) => { test.setTimeout(5 * 60 * 1000); // 5 minutes await login(page); await uploadLicense(page, expect); - await expect(page.locator('#app')).toContainText('Install in airgapped environmentt', { timeout: 15000 }); + await expect(page.locator('#app')).toContainText('Install in airgapped environment', { timeout: 15000 }); await page.getByText('download App Name from the Internet').click(); await expect(page.locator('#app')).toContainText('Installing your license'); await expect(page.locator('h3')).toContainText('My Example Config', { timeout: 30000 }); @@ -191,6 +191,6 @@ test('smoke test', async ({ page }) => { await page.locator('div').filter({ hasText: /^Change passwordAdd new applicationLog out$/ }).getByRole('img').click(); await page.getByText('Log out', { exact: true }).click(); await expect(page.getByPlaceholder('password')).toBeVisible({ timeout: 30000 }); - await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin console.'); + await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin consolee.'); await expect(page.getByRole('button')).toContainText('Log in'); }); From 78592b68b6aaaa3df30e431d185b14114dd925c4 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:41:57 +0000 Subject: [PATCH 6/7] fix test --- e2e/playwright/tests/smoke-test/test.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index f1a39ed62b..0bbb6b669f 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -191,6 +191,6 @@ test('smoke test', async ({ page }) => { await page.locator('div').filter({ hasText: /^Change passwordAdd new applicationLog out$/ }).getByRole('img').click(); await page.getByText('Log out', { exact: true }).click(); await expect(page.getByPlaceholder('password')).toBeVisible({ timeout: 30000 }); - await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin consolee.'); + await expect(page.locator('#app')).toContainText('Enter the password to access the App Name admin console.'); await expect(page.getByRole('button')).toContainText('Log in'); }); From 9c3e349fcfe5412f6ca782457808db573c07e9e9 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Thu, 25 Apr 2024 22:58:34 +0000 Subject: [PATCH 7/7] more time for registry settings --- e2e/playwright/tests/smoke-test/test.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/playwright/tests/smoke-test/test.spec.ts b/e2e/playwright/tests/smoke-test/test.spec.ts index 0bbb6b669f..b344ec442e 100644 --- a/e2e/playwright/tests/smoke-test/test.spec.ts +++ b/e2e/playwright/tests/smoke-test/test.spec.ts @@ -116,7 +116,7 @@ test('smoke test', async ({ page }) => { await expect(page.getByRole('button', { name: 'Save changes' })).toBeDisabled(); await expect(page.locator('.Loader')).toBeVisible(); await expect(page.locator('#app')).toContainText('Writing manifest to image destination', { timeout: 30000 }); - await expect(page.getByRole('button', { name: 'Save changes' })).toBeEnabled({ timeout: 30000 }); + await expect(page.getByRole('button', { name: 'Save changes' })).toBeEnabled({ timeout: 60000 }); await expect(page.locator('.Loader')).not.toBeVisible(); await page.getByRole('link', { name: 'Version history' }).click(); await expect(page.locator('#app')).toContainText('Registry Change');