diff --git a/.github/workflows/nodejs.yml b/.github/workflows/e2e.yml similarity index 75% rename from .github/workflows/nodejs.yml rename to .github/workflows/e2e.yml index 5bc597c..ffbbaae 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/e2e.yml @@ -1,9 +1,9 @@ -name: Gauge tests +name: Gauge e2e tests +# trigger e2e test run when pre-release is created on: - push: - branches: - - master + release: + types: [prerelease] jobs: test: @@ -30,7 +30,7 @@ jobs: npm ci - name: Run tests run: | - npm run automated + npm run automated | tee reports/logs/stdout.log env: CI: true SUITEST_TEST_PACK_ID: ${{ secrets.TEST_PACK_ID }} @@ -41,15 +41,8 @@ jobs: run: | curl -X POST -H "x-api-key:${{ secrets.CALLIOPE_TOKEN }}" -H "Content-Type:application/*" --data "@reports/xml-report/result.xml" "https://app.calliope.pro/api/v2/profile/764/report/import/junit" continue-on-error: true - - name: Archive jUnit results to artifacts + - name: Archive the reports and save them as artifact uses: actions/upload-artifact@v1 with: - name: jUnit report - path: reports/xml-report - continue-on-error: true - - name: Archive HTML results to artifacts - uses: actions/upload-artifact@v1 - with: - name: HTML report - path: reports/html-report - continue-on-error: true + name: Report + path: reports diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..689291d --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,49 @@ +name: Gauge smoke tests + +# trigger smoke test execution on every commit to master branch +on: + push: + branches: + - master + +jobs: + test: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: | + sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-keys 023EDB0B + echo deb https://dl.bintray.com/gauge/gauge-deb stable main | sudo tee -a /etc/apt/sources.list + sudo apt-get update + sudo apt-get install gauge + gauge install + npm ci + - name: Run tests + run: | + npm run automated:smoke | tee reports/logs/stdout.log + env: + CI: true + SUITEST_TEST_PACK_ID: ${{ secrets.TEST_PACK_ID }} + SUITEST_TOKEN_KEY: ${{ secrets.TOKEN_KEY }} + SUITEST_TOKEN_PASSWORD: ${{ secrets.TOKEN_PASSWORD }} + continue-on-error: true + - name: Report results to dashboard + run: | + curl -X POST -H "x-api-key:${{ secrets.CALLIOPE_TOKEN }}" -H "Content-Type:application/*" --data "@reports/xml-report/result.xml" "https://app.calliope.pro/api/v2/profile/765/report/import/junit" + continue-on-error: true + - name: Archive the reports and save them as artifact + uses: actions/upload-artifact@v1 + with: + name: Report + path: reports diff --git a/package.json b/package.json index 6977fba..09527dc 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,9 @@ "interactive:home": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, home\\\"", "interactive:video": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, video\\\"", "interactive:image": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, image\\\"", + "interactive:category": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, category\\\"", "interactive:smoke": "suitest interactive gauge run specs/ --simple-console --hide-suggestion --tags \\\"smoke\\\"", "automated": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e\\\"", - "automated:home": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, home\\\"", - "automated:video": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, video\\\"", - "automated:image": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"e2e, image\\\"", "automated:smoke": "suitest automated -t=$SUITEST_TEST_PACK_ID -k=$SUITEST_TOKEN_KEY -p=$SUITEST_TOKEN_PASSWORD gauge run specs/ --simple-console --hide-suggestion --tags \\\"smoke\\\"" }, "dependencies": { diff --git a/specs/app/category-test.spec b/specs/app/category-test.spec new file mode 100644 index 0000000..992855f --- /dev/null +++ b/specs/app/category-test.spec @@ -0,0 +1,23 @@ +# Category page test + Tags: e2e, category + +Test scenario makes sure that category page displays correct data. + +Illustrates how data can be fetched from server and used to build test scenarion dynamically. + +## Category page dynamic test + +First of all, we need to navigate to the page we would like to test. + +* Open application +* Press "up" +* Assert "WatchMe" navigation item is focused +* Press "right" exactly "3" times +* Press "enter" +* Assert element "First tile" exists + +Load the feed we would like to test against. Step is split in two: one to load data, another one to verify items on the screen +according to that data. This is done to illustrate how to work with dynamic items in Gauge. + +* Load "home-list-videos" page items +* Assert page items are rendered diff --git a/tests/application.js b/tests/application.js index 24f426b..8f7ebd7 100644 --- a/tests/application.js +++ b/tests/application.js @@ -1,5 +1,8 @@ /* globals gauge*/ "use strict"; +const fetch = require('node-fetch'); +const nodeAssert = require('assert'); + const suitest = require('suitest-js-api'); const {assert} = suitest; @@ -70,12 +73,17 @@ step("Make sure every main menu item is selectable", async () => { await isSelected(assert.element(getSelector(selector))).timeout(DEFAULT_TIMEOUT); // Wait until focus is moved to content tile in case it exists - if (await suitest.element(getSelector('First tile')).exists().timeout(DEFAULT_TIMEOUT)) { - // There is some contents, have to press up - await suitest.press(suitest.VRC.UP).until( - isFocused(suitest.element(getSelector(selector))) - ).repeat(3); + if (!await suitest.element(getSelector('First tile')).exists().timeout(DEFAULT_TIMEOUT)) { + // Log a message to the console - could be useful + console.log(`Top menu item "${selector}" does not have any data, skipping some steps`); + + continue; } + + // There is some contents, have to press up + await suitest.press(suitest.VRC.UP).until( + isFocused(suitest.element(getSelector(selector))) + ).repeat(3); } }); @@ -92,3 +100,37 @@ step("Run test ", async (testName) => { // Run pre-defined test editor test snippet await suitest.assert.runTest(getTestId(testName)); }); + +step("Load page items", async (page) => { + // Fetch data from server + const response = await fetch(`http://file.suite.st/sampleapp/api/${page}.json`); + + // ensure status code is 200. If it's something else + // - we have a problem already and it does not make sense to continue + nodeAssert.strictEqual(response.status, 200, 'Failed to load category data'); + + // wait for data to be loaded and JSON parsed + const data = await response.json(); + + // Save the data for use in other test lines + gauge.dataStore.scenarioStore.put('page-data', data); +}); + +step("Assert page items are rendered", async () => { + // Get currently loaded data from storage. Assume it was saved there in one of the previous steps + const data = gauge.dataStore.scenarioStore.get('page-data'); + + if (!data) { + // A precaution... + nodeAssert.fail('Data to test against was not recorded in storage. Forgot to load it?'); + } + + // For each item in data list + for (const item of data.list) { + // Make sure there is an element with item's name present in DOM + await assert.element({text: item.name}).exists().timeout(DEFAULT_TIMEOUT); + + // Make sure there is an image for given item and that it's loaded + await assert.element({css: `#listpageItem-img-${item.id}_img`}).matches(suitest.PROP.IMAGE_LOAD_STATE, suitest.IMAGE_LOAD_STATE.LOADED).timeout(DEFAULT_TIMEOUT); + } +});