Improve e2e browser test reliability (#1490)

Previously we were trying to start the webserver from playwright's globalSetup, which could have been causing our tests to hang. Now playwright will attempt to start the server, and timeout if that fails, rather than leaving the test in a running state. Further fixes include: - prevent dotenv being loading in CI environments, and being loaded multiple times in local environments (by moving it to globalSetup) - use npm link for testing against the current library build in CI - added a maximum timeout for e2e tests of 20 minutes (down from the default 360 minutes or 6 hours previously used) in CI * Improve E2E: Build once, test many: This allows us to build the code to test once, and then test it across many different operating systems. * Migrate browser e2e tests to use Github Environments: This should help give us clearer errors as to which environment is failing due to invalid credentials or other infrastructure issues. * Use a simple NextJS app for tests, and run basic browser-based tests on it * Update playwright config to use next test app * No longer send notification on failure Co-authored-by: Matthieu Bosquet <[email protected]> Co-authored-by: Nicolas Seydoux <[email protected]>
inrupt · Apr 11, 2022 · a2e1f01 · a2e1f01 · vercel · Apr 11, 2022
1 parent 8a8bce2
commit a2e1f01
Show file tree

Hide file tree

Showing 47 changed files with 9,587 additions and 458 deletions.
diff --git a/.codesandbox/sandbox/end-to-end-test.html b/.codesandbox/sandbox/end-to-end-test.html
diff --git a/.codesandbox/sandbox/package.json b/.codesandbox/sandbox/package.json
@@ -12,7 +12,9 @@
     "rdf-namespaces": "1.9.2"
   },
   "devDependencies": {
-    "parcel": "^2.0.0-beta.2"
+    "buffer": "^6.0.3",
+    "process": "^0.11.10",
+    "parcel": "^2.3.2"
   },
   "keywords": [
     "typescript",

diff --git a/.codesandbox/sandbox/src/end-to-end-test-helpers.ts b/.codesandbox/sandbox/src/end-to-end-test-helpers.ts
diff --git a/.codesandbox/sandbox/src/end-to-end-test.ts b/.codesandbox/sandbox/src/end-to-end-test.ts
diff --git a/.github/workflows/e2e-browser.yml b/.github/workflows/e2e-browser.yml
@@ -5,84 +5,104 @@ on:
 
 env:
   CI: true
+  DEBUG: "pw:webserver"
+  ACTIONS_STEP_DEBUG: true
 jobs:
-  test:
+  build:
+    if: github.actor != 'dependabot[bot]'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        id: setup-node
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Install dependencies
+        # FIXME: setup-node's caching seems to not restore correctly: https://github.com/actions/setup-node/pull/323/files
+        # if: steps.setup-node.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+
+      - name: Build package
+        run: npm run build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: build
+          path: dist
+
+  test-e2e-browser:
+    needs: build
+
     # Running end-to-end tests requires accessing secrets which aren't available to dependabot.
     # https://github.blog/changelog/2021-02-19-github-actions-workflows-triggered-by-dependabot-prs-will-run-with-read-only-permissions/
     if: github.actor != 'dependabot[bot]'
+    # Ensure we timeout reasonably quickly:
+    timeout-minutes: 30
+
     runs-on: ${{ matrix.os }}
+    environment:
+      name: ${{ matrix.environment-name }}
+
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        node-version: ["16"]
+        # Available OS's: https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
+        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        # The Node version does not influence how well our tests run in the browser,
+        # so we only need to test in one.
+        node-version: [16.x]
+        environment-name:
+          - "Inrupt Production"
+          - "Inrupt Dev-Next"
+
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+
+      - name: Download build artifact
+        uses: actions/download-artifact@v3
         with:
-          node-version: ${{ matrix.node-version }}
-      - name: Cache node modules
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-node-modules
+          name: build
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        id: setup-node
         with:
-          path: node_modules
-          key: ${{ runner.os }}-node${{ runner.node-version }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-      # Unfortunately `npm ci` regularly fails for reasons outside our control (e.g. network errors),
-      # so retry it twice if it fails to avoid those:
-      # (No, GitHub Actions at this point in time does not have native retry functionality.)
-      - run: npm ci || npm ci || npm ci
-      - run: npm run build
+          node-version: ${{ matrix.node-version }}
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Install dependencies
+        # FIXME: setup-node's caching seems to not restore correctly: https://github.com/actions/setup-node/pull/323/files
+        # if: steps.setup-node.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+      - name: Build the local code
+        run: npm run build
+
+      - name: Setup Playwright
+        run: npx playwright install --with-deps
+
       - name: Prepare browser-based end-to-end tests
-        run: |
-          cd .codesandbox/sandbox
-          npm install
-          # Install browsers
-          npx playwright install
-          npx playwright install-deps
-          # Run the end-to-end tests against the code in this branch specifically:
-          npm install ../../
-          cd ../..
+        run: npm run test:e2e:browser:build
+
       - name: Run browser-based end-to-end tests
         run: npm run test:e2e:browser
         env:
-          E2E_TEST_ESS_IDP_URL: ${{ secrets.E2E_TEST_ESS_PROD_IDP_URL }}
-          E2E_TEST_ESS_POD: ${{ secrets.E2E_TEST_ESS_PROD_POD }}
-          E2E_TEST_ESS_COGNITO_USER: ${{ secrets.E2E_TEST_ESS_PROD_COGNITO_USER }}
-          E2E_TEST_ESS_COGNITO_PASSWORD: ${{ secrets.E2E_TEST_ESS_PROD_COGNITO_PASSWORD }}
-          E2E_TEST_DEV_NEXT_POD: ${{ secrets.E2E_TEST_DEV_NEXT_POD }}
-          E2E_TEST_DEV_NEXT_IDP_URL: ${{ secrets.E2E_TEST_DEV_NEXT_IDP_URL }}
-          E2E_TEST_DEV_NEXT_COGNITO_USER: ${{ secrets.E2E_TEST_DEV_NEXT_COGNITO_USER }}
-          E2E_TEST_DEV_NEXT_COGNITO_PASSWORD: ${{ secrets.E2E_TEST_DEV_NEXT_COGNITO_PASSWORD }}
-      - name: Archive browser-based end-to-end test failure screenshots, if any
-        uses: actions/upload-artifact@v3
-        continue-on-error: true
-        if: failure()
-        with:
-          name: e2e-browser-failures
-          path: e2e-browser-failures
+          E2E_TEST_ENVIRONMENT: ${{ matrix.environment-name }}
+          E2E_TEST_POD: ${{ secrets.E2E_TEST_POD }}
+          E2E_TEST_IDP: ${{ secrets.E2E_TEST_IDP }}
+          E2E_TEST_USER: ${{ secrets.E2E_TEST_USER }}
+          E2E_TEST_PASSWORD: ${{ secrets.E2E_TEST_PASSWORD }}
+
       - name: Archive browser-based end-to-end test request logs
         uses: actions/upload-artifact@v3
         continue-on-error: true
         with:
           name: playwright-output
-          path: test-results/
-      - name: Archive code coverage results
-        uses: actions/upload-artifact@v3
-        continue-on-error: true
-        if: github.event_name != 'schedule'
-        with:
-          name: code-coverage-report
-          path: coverage
-      - name: Archive production artifacts
-        uses: actions/upload-artifact@v3
-        continue-on-error: true
-        if: github.event_name != 'schedule'
-        with:
-          name: dist
-          path: dist
-      - name: Send a notification that the test has failed
-        run: "curl -X POST -H Content-type: 'application/json' --data \"{\\\"text\\\":\\\"Automated tests against pod.inrupt.com and inrupt.net failed. View <https://github.com/inrupt/solid-client-js/actions/runs/$RUN_ID|the execution log> for more details.\\\"}\" $WEBHOOK_E2E_FAILURE"
-        if: failure() && github.event_name == 'schedule'
-        env:
-          WEBHOOK_E2E_FAILURE: "${{ secrets.WEBHOOK_E2E_FAILURE }}"
-          RUN_ID: "${{ github.run_id }}"
+          path: e2e/browser/test/test-results/
+        # Ensure test recordings are always uploaded if the exist:
+        if: ${{ always() }}
diff --git a/.github/workflows/e2e-node.yml b/.github/workflows/e2e-node.yml
@@ -16,14 +16,26 @@ jobs:
       name: ${{ matrix.environment-name }}
     strategy:
       matrix:
+        os: [ubuntu-latest]
+        node-version: [16.x]
         environment-name:
           ["Inrupt Production", "Inrupt Dev-Next", "Inrupt 1.1", "NSS"]
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+
+      - name: Runs on ${{ matrix.os }} with Node.js ${{ matrix.node-version }} against ${{ matrix.environment-name }}
+        uses: actions/setup-node@v3
+        id: setup-node
         with:
-          node-version: "16"
-      - run: npm ci --ignore-scripts
+          node-version: ${{ matrix.node-version }}
+          cache: "npm"
+          cache-dependency-path: "**/package-lock.json"
+
+      - name: Install dependencies
+        # FIXME: setup-node's caching seems to not restore correctly: https://github.com/actions/setup-node/pull/323/files
+        # if: steps.setup-node.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+
       - run: npm run test:e2e:node
         env:
           E2E_TEST_POD: ${{ secrets.E2E_TEST_POD }}

diff --git a/.gitignore b/.gitignore
@@ -14,4 +14,5 @@ docs/dist/
 .codesandbox/sandbox/.parcel-cache/
 test-results/
 .vercel
-.virtualenv
+.virtualenv
+.parcel-cache/
diff --git a/e2e/browser/.env.example b/e2e/browser/.env.example