Add playwright visual test using demo

.github/workflows/test.yml

@@ -14,10 +14,9 @@ defaults:
     shell: bash
 
 jobs:
-  test:
-    name: Build and test
+  lint:
+    name: Lint
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
@@ -32,22 +31,76 @@ jobs:
           init-shell: bash
           cache-downloads: true
 
-      - name: Install dependencies
+      - name: Lint
         run: |
           npm install
+          npm run lint:check
+
+  test:
+    name: End-to-end tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set conda environment
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-name: cockle
+          environment-file: environment-dev.yml
+          init-shell: bash
+          cache-downloads: true
 
       - name: Build
         run: |
+          npm install
           npm run build
 
-      - name: Lint
-        run: |
-          npm run lint:check
-
       - name: Run tests
         working-directory: test
         run: |
           npm install
           npx playwright install --with-deps chromium
           npm run build
           npm run test
+
+  ui-test:
+    name: Visual tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set conda environment
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-name: cockle
+          environment-file: environment-dev.yml
+          init-shell: bash
+          cache-downloads: true
+
+      - name: Build
+        run: |
+          npm install
+          npm run build
+
+      - name: Run visual tests
+        working-directory: demo
+        run: |
+          npm install
+          npx playwright install --with-deps chromium
+          npm run build
+          npm run test
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cockle-visual-tests
+          path: |
+            demo/test-results
+            demo/playwright-report

README.md

@@ -4,13 +4,12 @@ In-browser bash-like shell implemented in a combination of TypeScript and WebAss
 
 Used in the [JupyterLite terminal extension](https://github.com/jupyterlite/terminal).
 
-This is an early-stage work in progress and should be considered experimental code. Anything and
+⚠️ This is an early-stage work in progress and should be considered experimental code. Anything and
 everything could change at any time.
 
 The commands used here are either built-in commands implemented in TypeScript, or WebAssembly
-commands compiled into .js and .wasm files. The latter are built by
-[Emscripten-forge](https://emscripten-forge.org/) and are added to the `cockle` NPM package using
-a `micromamba` environment as part of the `npm prepack` process.
+commands compiled into `.js` and `.wasm` files. The latter are built by
+[Emscripten-forge](https://emscripten-forge.org/) and are added to a deployment during the build process.
 
 ## Build
 
@@ -22,19 +21,6 @@ npm run build
 npm run lint:check
 ```
 
-## Run tests
-
-```bash
-cd test
-npm install
-npx playwright install --with-deps chromium
-npm run build
-npm run test
-npm run test:report
-```
-
-You can interactively run individual tests using `npm run test:ui`.
-
 ## Demo
 
 The `cockle` repository includes a demo so that you can easily try it out interactively in a web
@@ -50,3 +36,23 @@ npm run serve
 then open a browser at the specified URL:
 
 <img alt="Demo" src="demo.png" width="500px">
+
+---
+
+## Testing
+
+The `test` directory contains playwright end-to-end tests which can be built and run as follows:
+
+```bash
+cd test
+npm install
+npx playwright install --with-deps chromium
+npm run build
+npm run test
+npm run test:report
+```
+
+You can interactively run individual tests using `npm run test:ui`.
+
+In addition, the `demo` directory contains separate visual tests that can be run in the same way.
+Only Linux screenshots are stored within the repository.

demo/package.json

@@ -10,10 +10,14 @@
     "scripts": {
         "build": "rspack build",
         "postbuild": "node node_modules/@jupyterlite/cockle/lib/tools/prepare_wasm.js --copy assets",
-        "serve": "rspack serve"
+        "serve": "rspack serve",
+        "test": "playwright test",
+        "test:ui": "playwright test --ui",
+        "test:report": "playwright show-report"
     },
     "devDependencies": {
         "@jupyterlite/cockle": "file:../",
+        "@playwright/test": "^1.45.3",
         "@rspack/cli": "^0.7.5",
         "@rspack/core": "^0.7.5",
         "@xterm/addon-fit": "^0.10.0",

demo/playwright.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './ui-tests',
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: 1,
+  reporter: [['html', { open: 'never' }]],
+  use: {
+    baseURL: 'http://localhost:4501',
+    trace: 'on-first-retry',
+    video: 'retain-on-failure'
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] }
+    }
+  ],
+  webServer: {
+    command: 'npm run serve',
+    url: 'http://localhost:4501',
+    reuseExistingServer: !process.env.CI
+  }
+});

demo/ui-tests/demo.test.ts

@@ -0,0 +1,40 @@
+import { test, expect } from '@playwright/test';
+
+test('visual test', async ({ page }) => {
+  await page.goto('/');
+
+  const wait = 100; // milliseconds
+
+  await page.locator('div.xterm-screen').click(); // sets focus for keyboard input
+
+  await page.keyboard.type('ls'); // avoid timestamps
+  await page.keyboard.press('Enter');
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.type('cp file.txt file2.txt');
+  await page.keyboard.press('Enter');
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.type('ls'); // avoid timestamps
+  await page.keyboard.press('Enter');
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.type('coc');
+  await page.keyboard.press('Tab'); // tab complete command name
+  await page.keyboard.press('Enter');
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.type('grep ember mon');
+  await page.keyboard.press('Tab'); // tab complete filename
+  await page.keyboard.press('Enter');
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.press('Tab'); // list all commands
+  await page.waitForTimeout(wait);
+
+  await page.keyboard.type('abc');
+  await page.keyboard.press('Enter'); // no such command
+  await page.waitForTimeout(wait);
+
+  await expect(page).toHaveScreenshot();
+});