diff --git a/src/agent/driver-test-runner.js b/src/agent/driver-test-runner.js index 5c2677b..56be474 100644 --- a/src/agent/driver-test-runner.js +++ b/src/agent/driver-test-runner.js @@ -180,7 +180,7 @@ export class DriverTestRunner { for (const assertion of test.assertions) { results.push({ command: command.id, - expectation: assertion.expectation, + expectation: assertion.expectation || assertion.assertionStatement, pass: true, }); } @@ -253,8 +253,13 @@ export class DriverTestRunner { export function validateKeysFromCommand(command) { const errors = []; for (let { id } of command.keypresses) { - // PAGE_DOWN and PAGE_UP are the only commands that have the extra _ inside a key - id = id.replace(/(PAGE)_(DOWN|UP)/, '$1$2'); + id = id + // PAGE_DOWN and PAGE_UP are the only commands that have the extra _ inside a key + .replace(/(PAGE)_(DOWN|UP)/, '$1$2') + // + is used to connect keys that are pressed simultaneously in v2 tests + .replace('+', '_') + // `UP_ARROW`, `DOWN_ARROW`, etc are sent as `up`, `down`, etc + .replace(/_ARROW/g, ''); if (/\//.test(id)) { errors.push(`'${id}' cannot contain '/'.`); } @@ -267,7 +272,7 @@ export function validateKeysFromCommand(command) { if (/\bfollowed\b/.test(id)) { errors.push(`'${id}' cannot contain 'followed' or 'followed by'.`); } - for (const part of id.split('_')) { + for (const part of id.split(/[_+,]/)) { // Some old test plans have keys that contain indications of unspecified // instructions ('/') or additional instructions that are not standardized // in test plans. These keys should be updated to be separate commands or @@ -296,6 +301,7 @@ export function atKeysFromCommand(command) { ATKey.chord( ...id .replace(/(PAGE)_(DOWN|UP)/, '$1$2') + .replace('+', '_') // + is used to connect keys that are pressed simultaneously in v2 tests .split('_') .map(key => key.trim().toLowerCase()) // `up arrow`, `down arrow`, etc are sent as `up`, `down`, etc diff --git a/src/agent/mock-test-runner.js b/src/agent/mock-test-runner.js index 1702ae0..6e8e31a 100644 --- a/src/agent/mock-test-runner.js +++ b/src/agent/mock-test-runner.js @@ -8,6 +8,7 @@ import { request } from 'http'; import { AgentMessage } from './messages.js'; +import { validateKeysFromCommand } from './driver-test-runner.js'; /** * @implements {AriaATCIAgent.TestRunner} @@ -51,19 +52,6 @@ export class MockTestRunner { this.log(AgentMessage.OPEN_PAGE, { url }); } - /** - * @param {CollectedTestCommand} command - * @param {CollectedTestAssertion} assertion - */ - async runAssertion(command, assertion) { - return { - command: command.id, - expectation: assertion.expectation, - output: `mocked output for ${assertion.expectation}`, - pass: await this.testAssertion(command, assertion), - }; - } - /** * @param {CollectedTestCommand} command * @param {CollectedTestAssertion} assertion @@ -82,6 +70,50 @@ export class MockTestRunner { this.baseUrl ) ); + + const commandsOutput = []; + const results = []; + + for (const command of task.commands) { + const { value: validCommand, errors } = validateKeysFromCommand(command); + if (validCommand) { + const mockOutput = `mocked output for ${command.id}`; + commandsOutput.push({ + command: validCommand.id, + output: mockOutput, + }); + + for (const assertion of task.assertions) { + const expectationText = assertion.expectation || assertion.assertionStatement; + + results.push({ + command: validCommand.id, + expectation: expectationText, + pass: await this.testAssertion(validCommand, assertion), + output: `mocked output for ${expectationText}`, + }); + } + } else { + await this.log(AgentMessage.INVALID_KEYS, { command, errors }); + + commandsOutput.push({ + command: command.id, + errors, + }); + + for (const assertion of task.assertions) { + const expectationText = assertion.expectation || assertion.assertionStatement; + + results.push({ + command: command.id, + expectation: expectationText, + output: `mocked output for ${expectationText}`, + pass: false, + }); + } + } + } + return { testId: task.info.testId, capabilities: { @@ -91,19 +123,8 @@ export class MockTestRunner { atVersion: '1.0', platformName: 'mock', }, - commands: await task.commands.reduce( - async (carry, command) => [ - ...(await carry), - ...(await task.assertions.reduce( - async (carry, assertion) => [ - ...(await carry), - await this.runAssertion(command, assertion), - ], - Promise.resolve([]) - )), - ], - Promise.resolve([]) - ), + commands: commandsOutput, + results, }; } } diff --git a/src/data/types.js b/src/data/types.js index 690cc7b..e80a7ff 100644 --- a/src/data/types.js +++ b/src/data/types.js @@ -41,7 +41,8 @@ * @property {string} [commands[].extraInstruction] human-readable additional instruction to follow * @property {object[]} assertions[] * @property {1 | 2} assertions[].priority - * @property {string} assertions[].expectation + * @property {string} [assertions[].expectation] assertion statement string, this property only exists on v1 tests + * @property {string} [assertions[].assertionStatement] assertion statement string, this property only exists on v2 tests */ /** diff --git a/src/host/tests/snapshots/agent.js.md b/src/host/tests/snapshots/agent.js.md index 26907a2..cfb461e 100644 --- a/src/host/tests/snapshots/agent.js.md +++ b/src/host/tests/snapshots/agent.js.md @@ -301,6 +301,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -365,6 +371,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -429,6 +441,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -472,6 +490,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -536,6 +560,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -600,6 +630,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -664,6 +700,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -707,6 +749,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -771,6 +819,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -815,6 +869,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -859,6 +919,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -903,6 +969,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -947,6 +1019,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -968,6 +1046,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1043,6 +1127,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1064,6 +1154,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1139,6 +1235,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1160,6 +1262,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1203,6 +1311,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1224,6 +1338,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1299,6 +1419,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1320,6 +1446,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1395,6 +1527,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1416,6 +1554,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1491,6 +1635,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1512,6 +1662,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1555,6 +1711,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1576,6 +1738,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1651,6 +1819,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1672,6 +1846,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1727,6 +1907,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1748,6 +1934,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1803,6 +1995,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1824,6 +2022,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', @@ -1879,6 +2083,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW', + output: 'mocked output for UP_ARROW', + }, + ], + results: [ { command: 'UP_ARROW', expectation: 'role up', @@ -1900,6 +2110,12 @@ Generated by [AVA](https://avajs.dev). platformName: 'mock', }, commands: [ + { + command: 'UP_ARROW,DOWN_ARROW', + output: 'mocked output for UP_ARROW,DOWN_ARROW', + }, + ], + results: [ { command: 'UP_ARROW,DOWN_ARROW', expectation: 'role down', diff --git a/src/host/tests/snapshots/agent.js.snap b/src/host/tests/snapshots/agent.js.snap index 13881eb..81806d0 100644 Binary files a/src/host/tests/snapshots/agent.js.snap and b/src/host/tests/snapshots/agent.js.snap differ diff --git a/src/host/tests/snapshots/cli-run-plan.js.md b/src/host/tests/snapshots/cli-run-plan.js.md index 7897e05..cd89c99 100644 --- a/src/host/tests/snapshots/cli-run-plan.js.md +++ b/src/host/tests/snapshots/cli-run-plan.js.md @@ -28,7 +28,7 @@ Generated by [AVA](https://avajs.dev). Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}]}␊ `, } @@ -56,7 +56,7 @@ Generated by [AVA](https://avajs.dev). Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -84,7 +84,7 @@ Generated by [AVA](https://avajs.dev). Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -114,15 +114,15 @@ Generated by [AVA](https://avajs.dev). `, stdout: `Callback Fetch Mocked: http://callback.url/ {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role up"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ Callback Fetch Mocked: http://callback.url/ {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role down"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -154,15 +154,15 @@ Generated by [AVA](https://avajs.dev). `, stdout: `Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role up"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ Callback Fetch Mocked: http://example.com/?TEST-STATUS=418 {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role down"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -194,14 +194,14 @@ Generated by [AVA](https://avajs.dev). `, stdout: `Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ method: 'post',␊ - body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role up"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ Callback Fetch Mocked: http://example.com/?TEST-STATUS=418&TEST-BAD-BODY {␊ method: 'post',␊ - body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for role down"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ + body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"responses":["mocked output for UP_ARROW,DOWN_ARROW"],"atVersionName":"1.0","browserVersionName":"1.0"}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","expectation":"role up","output":"mocked output for role up","pass":true}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","output":"mocked output for role down","pass":true}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } diff --git a/src/host/tests/snapshots/cli-run-plan.js.snap b/src/host/tests/snapshots/cli-run-plan.js.snap index 3218300..94fa60a 100644 Binary files a/src/host/tests/snapshots/cli-run-plan.js.snap and b/src/host/tests/snapshots/cli-run-plan.js.snap differ diff --git a/src/host/tests/snapshots/plan-from.js.snap b/src/host/tests/snapshots/plan-from.js.snap index 2b361cb..2e4b270 100644 Binary files a/src/host/tests/snapshots/plan-from.js.snap and b/src/host/tests/snapshots/plan-from.js.snap differ