diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f133aec7f21..25e99cee682e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,7 +62,7 @@ exclude_develop_master_rc: &exclude_develop_master_rc - /^Version-v(\d+)[.](\d+)[.](\d+)/ aliases: # Shallow Git Clone - - &shallow-git-clone + - &shallow-git-clone-and-enable-vnc name: Shallow Git Clone command: | #!/bin/bash @@ -88,6 +88,12 @@ aliases: git checkout -B "$CIRCLE_BRANCH" "$CIRCLE_SHA1" fi + # Piggyback on this alias to enable VNC connections + # The if statement will only be true on node-browsers executors + if [ "${SHELL}" != "/bin/bash" ]; then + cat ${HOME}/project/.circleci/scripts/enable-vnc.sh >> ~/.bashrc + fi + # Check if MMI Optional tests should run - &check-mmi-optional name: Check if MMI Optional tests should run @@ -376,7 +382,7 @@ jobs: trigger-beta-build: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -458,7 +464,7 @@ jobs: prep-deps: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - run: name: Save Yarn version @@ -495,7 +501,7 @@ jobs: get-changed-files-with-git-diff: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -510,7 +516,7 @@ jobs: validate-locales-only: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -519,7 +525,7 @@ jobs: validate-lavamoat-allow-scripts: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -533,7 +539,7 @@ jobs: validate-lavamoat-policy-build: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -550,7 +556,7 @@ jobs: build-type: type: string steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -564,7 +570,7 @@ jobs: prep-build: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -601,7 +607,7 @@ jobs: prep-build-mv2: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -644,7 +650,7 @@ jobs: prep-build-mmi: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -688,7 +694,7 @@ jobs: prep-build-flask: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -729,7 +735,7 @@ jobs: prep-build-flask-mv2: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -770,7 +776,7 @@ jobs: prep-build-test-flask: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -792,7 +798,7 @@ jobs: prep-build-test-flask-mv2: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -814,7 +820,7 @@ jobs: prep-build-test-mmi: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -836,7 +842,7 @@ jobs: prep-build-test-mmi-playwright: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -865,7 +871,7 @@ jobs: prep-build-test: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -889,7 +895,7 @@ jobs: prep-build-test-mv2: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -913,7 +919,7 @@ jobs: prep-build-test-webpack: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - attach_workspace: at: . - run: @@ -933,7 +939,7 @@ jobs: prep-build-storybook: executor: node-linux-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -948,7 +954,7 @@ jobs: prep-build-ts-migration-dashboard: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -963,7 +969,7 @@ jobs: test-yarn-dedupe: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -974,7 +980,7 @@ jobs: test-lint: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -988,7 +994,7 @@ jobs: test-storybook: executor: node-browsers-medium-plus steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1011,7 +1017,7 @@ jobs: test-lint-lockfile: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1025,7 +1031,7 @@ jobs: test-lint-changelog: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1052,7 +1058,7 @@ jobs: test-deps-audit: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1063,7 +1069,7 @@ jobs: test-deps-depcheck: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1075,7 +1081,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 20 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1095,7 +1101,7 @@ jobs: test-api-specs: executor: node-browsers-medium-plus steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1127,7 +1133,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 20 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1151,7 +1157,7 @@ jobs: executor: node-browsers-medium parallelism: 1 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1175,7 +1181,7 @@ jobs: executor: node-browsers-small parallelism: 1 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1198,7 +1204,7 @@ jobs: test-e2e-chrome-rpc-mmi: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1221,7 +1227,7 @@ jobs: test-e2e-chrome-vault-decryption: executor: node-browsers-medium-plus steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1239,7 +1245,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 10 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1263,7 +1269,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 8 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1287,7 +1293,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 12 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1311,7 +1317,7 @@ jobs: executor: playwright parallelism: 2 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -1352,7 +1358,7 @@ jobs: executor: playwright parallelism: 2 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: corepack enable - attach_workspace: at: . @@ -1382,7 +1388,7 @@ jobs: executor: node-browsers-medium-plus parallelism: 24 steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1405,7 +1411,7 @@ jobs: benchmark: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1429,7 +1435,7 @@ jobs: user-actions-benchmark: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1453,7 +1459,7 @@ jobs: stats-module-load-init: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1557,7 +1563,7 @@ jobs: job-publish-release: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1586,7 +1592,7 @@ jobs: - add_ssh_keys: fingerprints: - '3d:49:29:f4:b2:e8:ea:af:d1:32:eb:2a:fc:15:85:d8' - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1602,7 +1608,7 @@ jobs: - add_ssh_keys: fingerprints: - '8b:21:e3:20:7c:c9:db:82:74:2d:86:d6:11:a7:2f:49' - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1617,7 +1623,7 @@ jobs: validate-source-maps: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1628,7 +1634,7 @@ jobs: validate-source-maps-beta: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1639,7 +1645,7 @@ jobs: validate-source-maps-mmi: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1656,7 +1662,7 @@ jobs: validate-source-maps-flask: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1673,7 +1679,7 @@ jobs: validate-source-maps-flask-mv2: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1690,7 +1696,7 @@ jobs: validate-source-maps-mv2: executor: node-browsers-small steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1707,7 +1713,7 @@ jobs: test-mozilla-lint-mv2: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . @@ -1724,7 +1730,7 @@ jobs: test-mozilla-lint-flask-mv2: executor: node-browsers-medium steps: - - run: *shallow-git-clone + - run: *shallow-git-clone-and-enable-vnc - run: sudo corepack enable - attach_workspace: at: . diff --git a/.circleci/scripts/enable-vnc.sh b/.circleci/scripts/enable-vnc.sh new file mode 100644 index 000000000000..164334289c8c --- /dev/null +++ b/.circleci/scripts/enable-vnc.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# This script is based on the documentation from CircleCI, which does not work as written +# https://circleci.com/docs/browser-testing/#interacting-with-the-browser-over-vnc + +set -e +set -u +set -o pipefail +set -x + +cd "${HOME}/project" + +# Install a VNC server +readonly LOCK_FILE="installed.lock" +if [ ! -f "${LOCK_FILE}" ]; then + sudo apt update + sudo apt install -y x11vnc + + touch "${LOCK_FILE}" +fi + +# Start VNC server +if ! pgrep x11vnc > /dev/null; then + x11vnc -display "$DISPLAY" -bg -forever -nopw -quiet -listen localhost -xkb -rfbport 5901 -passwd password +fi diff --git a/.circleci/scripts/test-run-e2e.sh b/.circleci/scripts/test-run-e2e.sh index 25ae1eb7b64c..5d71a6f561b0 100755 --- a/.circleci/scripts/test-run-e2e.sh +++ b/.circleci/scripts/test-run-e2e.sh @@ -11,11 +11,20 @@ then exit 1 fi +# Skip running e2e tests if we're doing "Rerun job with SSH" and we're not on the first node +if netstat -tnlp | grep -q 'circleci-agent' && [ "$CIRCLE_NODE_INDEX" -ne 0 ] +then + printf '"Rerun job with SSH" and not on the first node, shutting down\n' + circleci-agent step halt + exit 1 +fi + # Run the actual test command from the parameters timeout 20m "$@" --retries 1 # Error code 124 means the command timed out -if [ $? -eq 124 ]; then +if [ $? -eq 124 ] +then # Once deleted, if someone tries to "Rerun failed tests" the result will be # "Error: can not rerun failed tests: no failed tests could be found" echo 'Timeout error, deleting the test results' diff --git a/docs/ssh-to-circleci.md b/docs/ssh-to-circleci.md new file mode 100644 index 000000000000..03a1a283aa99 --- /dev/null +++ b/docs/ssh-to-circleci.md @@ -0,0 +1,33 @@ +# Debugging E2E tests by connecting to CircleCI over SSH and VNC + +Developers often say "I can't reproduce this CI failure locally, but it fails on CircleCI." + +If you find yourself in that situation, one option is to use Codespaces, which can reproduce some of these failures, and is a little bit easier to use. + +The other NEW option is to SSH into CircleCI and use VNC. + +1. You must be logged into CircleCI and have access to metamask-extension +2. "Rerun job with SSH" [Documentation](https://circleci.com/docs/ssh-access-jobs/) +3. Look for this instruction inside the `Enable SSH` section + + ``` + You can now SSH into this box if your SSH public key is added: + $ ssh -p xxxxx xxx.xxx.xxx.xxx + ``` + +4. Copy the command that CircleCI gives you and append `-L 5901:localhost:5901` (this will tunnel the VNC connection over SSH) +5. Enter this in a terminal, for example `ssh -p xxxxx xxx.xxx.xxx.xxx -L 5901:localhost:5901` +6. When you login to SSH, it automatically executes `/.circleci/scripts/enable-vnc.sh` to set up the connection +7. Use your favorite VNC viewer on your local machine to connect to `localhost:5901` + - Mac: In the Terminal, run `open vnc://localhost:5901` + - Linux: `tigervnc-viewer` is a good package to match the server + - Windows: [RealVNC Viewer](https://www.realvnc.com/en/connect/download/viewer/windows/) or [TightVNC](https://www.tightvnc.com/download.php) +8. The VNC password is simply `password` +9. The normal E2E tests will already be running, and you can watch them run +10. If you want to stop the normal tests and run your own tests, run `pkill timeout` (this works because .circleci/scripts/test-run-e2e.sh runs the `timeout` command) + +### Notes: + +- This procedure was based on the documentation from CircleCI [here](https://circleci.com/docs/browser-testing/#interacting-with-the-browser-over-vnc). The way they wrote it doesn't really work correctly, but we fixed it. +- If you run "Rerun job with SSH" on a job that has `parallelism: 24`, it will rerun all 24 VMs, but quickly shut down all but the first one. +- **Advanced Usage:** If you don't want to run the `.bashrc` when you connect, append this to the SSH command `-t bash --norc --noprofile`