Split integration to smoke and e2e tests, combined all logs into sing…

…le archive

.github/workflows/nodejs.yml → .github/workflows/e2e.yml

@@ -1,9 +1,9 @@
-name: Gauge tests
+name: Gauge e2e tests
 
+# trigger e2e test run when pre-release is created
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [prerelease]
 
 jobs:
   test:
@@ -30,7 +30,7 @@ jobs:
         npm ci
     - name: Run tests
       run: |
-        npm run automated
+        npm run automated | tee reports/logs/stdout.log
       env:
         CI: true
         SUITEST_TEST_PACK_ID: ${{ secrets.TEST_PACK_ID }}
@@ -41,15 +41,8 @@ jobs:
       run: |
         curl -X POST -H "x-api-key:${{ secrets.CALLIOPE_TOKEN }}" -H "Content-Type:application/*" --data "@reports/xml-report/result.xml" "https://app.calliope.pro/api/v2/profile/764/report/import/junit"
       continue-on-error: true
-    - name: Archive jUnit results to artifacts
+    - name: Archive the reports and save them as artifact
       uses: actions/upload-artifact@v1
       with:
-        name: jUnit report
-        path: reports/xml-report
-      continue-on-error: true
-    - name: Archive HTML results to artifacts
-      uses: actions/upload-artifact@v1
-      with:
-        name: HTML report
-        path: reports/html-report
-      continue-on-error: true
+        name: Report
+        path: reports

.github/workflows/smoke.yml

@@ -0,0 +1,49 @@
+name: Gauge smoke tests
+
+# trigger smoke test execution on every commit to master branch
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [12.x]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependencies
+      run: |
+        sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-keys 023EDB0B
+        echo deb https://dl.bintray.com/gauge/gauge-deb stable main | sudo tee -a /etc/apt/sources.list
+        sudo apt-get update
+        sudo apt-get install gauge
+        gauge install
+        npm ci
+    - name: Run tests
+      run: |
+        npm run automated:smoke | tee reports/logs/stdout.log
+      env:
+        CI: true
+        SUITEST_TEST_PACK_ID: ${{ secrets.TEST_PACK_ID }}
+        SUITEST_TOKEN_KEY: ${{ secrets.TOKEN_KEY }}
+        SUITEST_TOKEN_PASSWORD: ${{ secrets.TOKEN_PASSWORD }}
+      continue-on-error: true
+    - name: Report results to dashboard
+      run: |
+        curl -X POST -H "x-api-key:${{ secrets.CALLIOPE_TOKEN }}" -H "Content-Type:application/*" --data "@reports/xml-report/result.xml" "https://app.calliope.pro/api/v2/profile/765/report/import/junit"
+      continue-on-error: true
+    - name: Archive the reports and save them as artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: Report
+        path: reports

package.json

@@ -7,11 +7,9 @@
     "interactive:home": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, home\\\"",
     "interactive:video": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, video\\\"",
     "interactive:image": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, image\\\"",
+    "interactive:category": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, category\\\"",
     "interactive:smoke": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"smoke\\\"",
     "automated": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e\\\"",
-    "automated:home": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, home\\\"",
-    "automated:video": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, video\\\"",
-    "automated:image": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, image\\\"",
     "automated:smoke": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"smoke\\\""
   },
   "dependencies": {

specs/app/category-test.spec

@@ -0,0 +1,23 @@
+# Category page test
+    Tags: e2e, category
+
+Test scenario makes sure that category page displays correct data.
+
+Illustrates how data can be fetched from server and used to build test scenarion dynamically.
+
+## Category page dynamic test
+
+First of all, we need to navigate to the page we would like to test.
+
+* Open application
+* Press "up"
+* Assert "WatchMe" navigation item is focused
+* Press "right" exactly "3" times
+* Press "enter"
+* Assert element "First tile" exists
+
+Load the feed we would like to test against. Step is split in two: one to load data, another one to verify items on the screen
+according to that data. This is done to illustrate how to work with dynamic items in Gauge.
+
+* Load "home-list-videos" page items
+* Assert page items are rendered

tests/application.js

@@ -1,5 +1,8 @@
 /* globals gauge*/
 "use strict";
+const fetch = require('node-fetch');
+const nodeAssert = require('assert');
+
 const suitest = require('suitest-js-api');
 const {assert} = suitest;
 
@@ -70,12 +73,17 @@ step("Make sure every main menu item is selectable", async () => {
         await isSelected(assert.element(getSelector(selector))).timeout(DEFAULT_TIMEOUT);
 
         // Wait until focus is moved to content tile in case it exists
-        if (await suitest.element(getSelector('First tile')).exists().timeout(DEFAULT_TIMEOUT)) {
-            // There is some contents, have to press up
-            await suitest.press(suitest.VRC.UP).until(
-                isFocused(suitest.element(getSelector(selector)))
-            ).repeat(3);
+        if (!await suitest.element(getSelector('First tile')).exists().timeout(DEFAULT_TIMEOUT)) {
+            // Log a message to the console - could be useful
+            console.log(`Top menu item "${selector}" does not have any data, skipping some steps`);
+
+            continue;
         }
+
+        // There is some contents, have to press up
+        await suitest.press(suitest.VRC.UP).until(
+            isFocused(suitest.element(getSelector(selector)))
+        ).repeat(3);
     }
 });
 
@@ -92,3 +100,37 @@ step("Run test <testName>", async (testName) => {
     // Run pre-defined test editor test snippet
     await suitest.assert.runTest(getTestId(testName));
 });
+
+step("Load <page> page items", async (page) => {
+    // Fetch data from server
+    const response = await fetch(`http://file.suite.st/sampleapp/api/${page}.json`);
+
+    // ensure status code is 200. If it's something else
+    // - we have a problem already and it does not make sense to continue
+    nodeAssert.strictEqual(response.status, 200, 'Failed to load category data');
+
+    // wait for data to be loaded and JSON parsed
+    const data = await response.json();
+
+    // Save the data for use in other test lines
+    gauge.dataStore.scenarioStore.put('page-data', data);
+});
+
+step("Assert page items are rendered", async () => {
+    // Get currently loaded data from storage. Assume it was saved there in one of the previous steps
+    const data = gauge.dataStore.scenarioStore.get('page-data');
+
+    if (!data) {
+        // A precaution...
+        nodeAssert.fail('Data to test against was not recorded in storage. Forgot to load it?');
+    }
+
+    // For each item in data list
+    for (const item of data.list) {
+        // Make sure there is an element with item's name present in DOM
+        await assert.element({text: item.name}).exists().timeout(DEFAULT_TIMEOUT);
+
+        // Make sure there is an image for given item and that it's loaded
+        await assert.element({css: `#listpageItem-img-${item.id}_img`}).matches(suitest.PROP.IMAGE_LOAD_STATE, suitest.IMAGE_LOAD_STATE.LOADED).timeout(DEFAULT_TIMEOUT);
+    }
+});