Attach / upload Playwright report on failure (#4573)

.github/actions/kots-e2e/action.yml

@@ -123,6 +123,12 @@ runs:
           SKIP_TEARDOWN=1
       shell: bash
 
+    - uses: actions/upload-artifact@v4
+      if: ${{ failure() }}
+      with:
+        name: ${{ github.job }}-${{ inputs.k8s-distribution }}-${{ inputs.k8s-version }}-playwright-report
+        path: ./e2e/playwright/playwright-report/
+
     - name: Print logs on failure
       if: ${{ failure() }}
       env:

e2e/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 

e2e/Makefile

@@ -2,6 +2,7 @@ include ../Makefile.build.mk
 
 BIN_DIR := $(shell pwd)/bin
 KOTS_BIN_DIR := $(shell dirname $(shell pwd))/bin
+PLAYWRIGHT_DIR := $(shell pwd)/playwright
 
 SHELL := /bin/bash
 
@@ -32,6 +33,7 @@ test:
 		-v $(BIN_DIR)/e2e.test:/usr/local/bin/e2e.test \
 		-v $(KOTS_BIN_DIR)/kots:/usr/local/bin/kots \
 		-v $(KOTS_BIN_DIR)/kots:/usr/local/bin/kubectl-kots \
+		-v $(PLAYWRIGHT_DIR)/playwright-report:/playwright/playwright-report \
 		$(EXISTING_KUBECONFIG_VOLUME_MOUNT) \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		e2e-deps \

e2e/playwright/playwright.config.ts

@@ -11,14 +11,11 @@ import { defineConfig, devices } from '@playwright/test';
  */
 export default defineConfig({
   testDir: './tests',
-  /* Run tests in files in parallel */
-  fullyParallel: true,
-  /* Fail the build on CI if you accidentally left test.only in the source code. */
-  forbidOnly: !!process.env.CI,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'list',
+  reporter: [
+    ['line'],
+    ['html', { open: 'never' }],
+  ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */

e2e/playwright/tests/smoke-test/test.spec.ts

@@ -21,7 +21,7 @@ test('smoke test', async ({ page }) => {
   await page.getByRole('button', { name: 'Deploy' }).click();
   await page.getByRole('button', { name: 'Deploy anyway' }).click();
   await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 });
-  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 15000 });
+  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 30000 });
   await expect(page.locator('#app')).toContainText('Check for update');
   await expect(page.locator('#app')).toContainText('Configure automatic updates');
   await expect(page.locator('#app')).toContainText('Redeploy', { timeout: 15000 });
@@ -41,6 +41,7 @@ test('smoke test', async ({ page }) => {
   await expect(page.locator('.ConfigureUpdatesModal')).toContainText('At 12:00 AM, only on Sunday');
   await page.getByRole('button', { name: 'Update', exact: true }).click();
   await expect(page.getByText('Automatically check for updates', { exact: true })).not.toBeVisible();
+  await page.waitForTimeout(2000);
   await page.locator('span[data-tip="View deploy logs"]').first().click();
   await validateDeployLogs(page, expect);
   await page.reload();
@@ -115,7 +116,7 @@ test('smoke test', async ({ page }) => {
   await expect(page.getByRole('button', { name: 'Save changes' })).toBeDisabled();
   await expect(page.locator('.Loader')).toBeVisible();
   await expect(page.locator('#app')).toContainText('Writing manifest to image destination', { timeout: 30000 });
-  await expect(page.getByRole('button', { name: 'Save changes' })).toBeEnabled({ timeout: 30000 });
+  await expect(page.getByRole('button', { name: 'Save changes' })).toBeEnabled({ timeout: 60000 });
   await expect(page.locator('.Loader')).not.toBeVisible();
   await page.getByRole('link', { name: 'Version history' }).click();
   await expect(page.locator('#app')).toContainText('Registry Change');