E2E tests: run against atomic test site (#20807)

* * Add config for atomic test site * Always login in wpcom in e2e tests relevant for atomic * Github workflow for e2e tests on atomic * Add changelog * Add push event for testing * Add missing workflow step * Add empty browserlist to fix babel error * Install Jetpack CLI * Merge install steps * Temporary skipping wordads test on atomic run * Disable push trigger
Automattic · Aug 27, 2021 · 323a331 · 323a331
1 parent b5f4bd4
commit 323a331
Show file tree

Hide file tree

Showing 6 changed files with 176 additions and 2 deletions.
diff --git a/.github/workflows/e2e-tests-atomic.yml b/.github/workflows/e2e-tests-atomic.yml
@@ -0,0 +1,168 @@
+name: E2E Tests on Atomic sites
+
+on:
+  schedule:
+    - cron:  '0 */4 * * *'
+
+jobs:
+  e2e-tests:
+    name: "E2E tests"
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Read tool versions
+      id: versions
+      uses: ./.github/actions/read-versions
+
+    - name: Setup Node
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ steps.versions.outputs.node-version }}
+
+    - name: Use pnpm cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.pnpm-store
+        key: ${{ runner.os }}-pnpm-${{ steps.versions.outputs.node-version }}-${{ hashFiles('**/package.json') }}
+        restore-keys: |
+          ${{ runner.os }}-pnpm-${{ steps.versions.outputs.node-version }}-
+
+    - name: Setup pnpm
+      uses: pnpm/[email protected]
+      with:
+        version: ${{ steps.versions.outputs.pnpm-version }}
+
+    - name: Tool versions
+      run: |
+        which node
+        node --version
+        which pnpm
+        pnpm --version
+        which jq
+        jq --version
+
+    - name: Install
+      working-directory: projects/plugins/jetpack/tests/e2e
+      env:
+        CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }}
+      run: |
+        pnpm install --dir ../../../../tools/cli
+        pnpm install
+        pnpm run test-decrypt-config
+
+    - name: Run tests
+      working-directory: projects/plugins/jetpack/tests/e2e
+      env:
+        TEST_SITE: atomic
+      run: pnpm run test-e2e -- --group=atomic
+
+    - name: Upload test artifacts
+      if: ${{ always() }}
+      continue-on-error: true
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-output-${{ matrix.group }}
+        path: projects/plugins/jetpack/tests/e2e/output
+
+    - name: Send Slack notification
+      if: ${{ failure() }}
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      working-directory: projects/plugins/jetpack/tests/e2e
+      run: NODE_ENV=test node ./bin/slack.js run atomic
+
+    - uses: testspace-com/setup-testspace@v1
+      with:
+        domain: ${{github.repository_owner}}
+      if: always()
+    - name: Publish to Testspace
+      working-directory: projects/plugins/jetpack/tests/e2e
+      run: |
+        testspace "[${GROUP}]output/reports/junit-*.xml"
+      if: always()
+
+  slack-notification:
+    name: "Slack notification"
+    runs-on: ubuntu-latest
+    needs: e2e-tests
+    env:
+      CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }}
+      GITHUB_CONTEXT: ${{ toJson(github) }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Read tool versions
+        id: versions
+        uses: ./.github/actions/read-versions
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ steps.versions.outputs.node-version }}
+
+      - name: Use pnpm cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.pnpm-store
+          key: ${{ runner.os }}-pnpm-${{ steps.versions.outputs.node-version }}-${{ hashFiles('**/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-${{ steps.versions.outputs.node-version }}-
+
+      - name: Setup pnpm
+        uses: pnpm/[email protected]
+        with:
+          version: ${{ steps.versions.outputs.pnpm-version }}
+
+      - name: Tool versions
+        run: |
+          which node
+          node --version
+          which pnpm
+          pnpm --version
+
+      - name: Send Slack notification
+        working-directory: projects/plugins/jetpack/tests/e2e
+        env:
+          RESULT: ${{ needs.e2e-tests.result }}
+        run: |
+          pnpm install
+          pnpm run test-decrypt-config
+          NODE_ENV=test node ./bin/slack.js job $RESULT
+
+  test-reports:
+    name: "Trigger test report workflow"
+    runs-on: ubuntu-latest
+    if: ${{ ! cancelled() }}
+    needs: e2e-tests
+
+    steps:
+      - name: Trigger test report workflow
+        env:
+          TOKEN: ${{ secrets.E2E_TEST_REPORTS_TOKEN }}
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
+          	BRANCH=$GITHUB_HEAD_REF
+            EVENT_NAME="$PR_NUMBER: $PR_TITLE"
+          else
+          	BRANCH=${GITHUB_REF:11}
+            EVENT_NAME="Run $GITHUB_RUN_ID"
+          fi
+
+          curl -X POST https://api.github.com/repos/automattic/jetpack-e2e-reports/dispatches \
+          -H "Accept: application/vnd.github.v3+json" \
+          -u "user:$TOKEN" \
+          --data "{\"event_type\": \"$EVENT_NAME\",
+          \"client_payload\": {
+          \"repository\": \"$GITHUB_REPOSITORY\",
+          \"run_id\": \"$GITHUB_RUN_ID\",
+          \"run_number\": \"$GITHUB_RUN_NUMBER\",
+          \"branch\": \"$BRANCH\",
+          \"pr_title\": \"$PR_TITLE\",
+          \"pr_number\": \"$PR_NUMBER\"
+          }}"
diff --git a/projects/plugins/jetpack/changelog/e2e-atomic-e2e-ci b/projects/plugins/jetpack/changelog/e2e-atomic-e2e-ci
@@ -0,0 +1,4 @@
+Significance: minor
+Type: enhancement
+
+E2E tests: scheduled test runs on atomic test site
diff --git a/projects/plugins/jetpack/tests/e2e/config/encrypted.enc b/projects/plugins/jetpack/tests/e2e/config/encrypted.enc
diff --git a/projects/plugins/jetpack/tests/e2e/package.json b/projects/plugins/jetpack/tests/e2e/package.json
@@ -59,5 +59,6 @@
 		"node": "^14.16.1",
 		"pnpm": "^6.5.0",
 		"yarn": "use pnpm instead - see docs/yarn-upgrade.md"
-	}
+	},
+	"browserslist": []
 }
diff --git a/projects/plugins/jetpack/tests/e2e/specs/free-blocks.test.js b/projects/plugins/jetpack/tests/e2e/specs/free-blocks.test.js
@@ -21,6 +21,7 @@ describe( 'Free blocks', () => {
 
 	beforeAll( async () => {
 		await prerequisitesBuilder()
+			.withWpComLoggedIn( true )
 			.withLoggedIn( true )
 			.withConnection( true )
 			.withPlan( Plans.Free )

diff --git a/projects/plugins/jetpack/tests/e2e/specs/pro-blocks.test.js b/projects/plugins/jetpack/tests/e2e/specs/pro-blocks.test.js
@@ -20,8 +20,8 @@ describe( 'Paid blocks', () => {
 
 	beforeAll( async () => {
 		await prerequisitesBuilder()
-			.withLoggedIn( true )
 			.withWpComLoggedIn( true )
+			.withLoggedIn( true )
 			.withConnection( true )
 			.withPlan( Plans.Complete )
 			.withActiveModules( [ 'publicize' ] )