diff --git a/README.md b/README.md index aeaea77..d5a2593 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,10 @@ Firefox, execute the following command in a terminal: $ node aria-at-automation-harness/bin/host.js run-plan \ --plan-workingdir aria-at/build/tests/horizontal-slider \ '{reference/**,test-*-nvda.*}' \ - --agent-web-driver-url=http://127.0.0.1:4444 \ - --agent-at-driver-url=ws://127.0.0.1:4382/session \ + --web-driver-url=http://127.0.0.1:4444 \ + --at-driver-url=ws://127.0.0.1:4382/session \ --reference-hostname=127.0.0.1 \ - --agent-web-driver-browser=firefox + --web-driver-browser=firefox ``` ## [aria-at-automation](https://github.com/w3c/aria-at-automation) diff --git a/bin/agent.js b/bin/agent.js deleted file mode 100755 index e448461..0000000 --- a/bin/agent.js +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env node - -import { runCli } from '../src/agent/index.js'; -runCli(); diff --git a/package.json b/package.json index 3bceac3..b0fbd90 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "test": "npm run test:types && npm run test:unit", "test:types": "tsc", "test:unit": "ava", + "test:update": "ava --update-snapshots", "postinstall": "husky install" }, "lint-staged": { diff --git a/src/agent/README.md b/src/agent/README.md deleted file mode 100644 index 6a4de10..0000000 --- a/src/agent/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# `src/agent` - -This directory implements the systems used by the `bin/agent.js` command line tool. - -At this time invoking this command must be done from a nodejs process with child_process.fork. A developer tool may be provided soon to read a single test file run `agent` and write the result to console output. - -```sh -$ bin/agent.js -agent.js - -Run tests from input - -Options: -... - -Error: Currently, this command may only be used when launched by a nodejs child_process.fork call. -``` - -Developing with `bin/agent.js` you may want to use the mock test runner. Calling `agent.js` with `--mock` will enable the mock test runner. The mock test runner will be removed in the near future and replaced with the ability to create a mock server that can be more tailored to individual mock use cases. - -## main command - -``` -$ bin/agent.js --help --show-hidden -agent.js - -Run tests from input - -Options: - --help Show help [boolean] - --version Show version number [boolean] - --quiet Disable all logging - --debug Enable all logging - --verbose Enable a subset of logging messages - --reference-base-url Url to append reference page listed in tests to - [string] [default: "http://localhost:8000"] - --web-driver-url [default: "http://localhost:4444"] - --web-driver-browser [choices: "chrome", "firefox"] [default: "firefox"] - --at-driver-url [default: "ws://localhost:4382"] - --mock [boolean] - --mock-open-page [choices: "request", "skip"] -``` - -Currently this command must be executed by a parent node process as a node fork child process. - -### `--verbose` options - -The main command's verbose level can be set with `--debug`, `--quiet`, or `--verbose`. `--verbose` takes a comma separate list of the following logging message types. - -- `start` -- `uncaughtError` -- `willStop` -- `startTest` -- `openPage` -- `invalidKeys` -- `pressKeys` -- `speechEvent` -- `noRunTestSetup` diff --git a/src/agent/cli.js b/src/agent/cli.js deleted file mode 100644 index 66f116e..0000000 --- a/src/agent/cli.js +++ /dev/null @@ -1,352 +0,0 @@ -/// - -/** - * @module agent - */ - -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; - -import { iterateEmitter } from '../shared/iterate-emitter.js'; - -import { createRunner } from './create-test-runner.js'; -import { agentMain } from './main.js'; -import { AgentMessage, createAgentLogger } from './messages.js'; -import { getTimesOption, timesArgs, timesOptionsConfig } from '../shared/times-option.js'; - -/** @param {yargs} args */ -export function buildAgentCliOptions(args = yargs) { - return args - .options({ - quiet: { - conflicts: ['debug', 'verbose'], - describe: 'Disable all logging', - }, - debug: { - conflicts: ['quiet', 'verbose'], - describe: 'Enable all logging', - }, - verbose: { - coerce(arg) { - if (!arg) { - return; - } - const messageValues = Object.values(AgentMessage); - const verbosity = arg.split(','); - for (const name of verbosity) { - if (!messageValues.includes(name)) { - throw new Error( - `--verbose must be a comma separated list including: ${messageValues.join(', ')}` - ); - } - } - return verbosity; - }, - conflicts: ['debug', 'quiet'], - describe: 'Enable a subset of logging messages', - nargs: 1, - }, - 'reference-base-url': { - description: 'Url to append reference page listed in tests to', - coerce(arg) { - return new URL(arg); - }, - default: 'http://localhost:8000', - }, - 'web-driver-url': { - coerce(arg) { - return new URL(arg); - }, - default: 'http://localhost:4444', - }, - 'web-driver-browser': { - choices: ['chrome', 'firefox', 'safari'], - default: 'firefox', - }, - 'at-driver-url': { - coerce(arg) { - return new URL(arg); - }, - default: 'ws://localhost:4382', - }, - mock: { - type: 'boolean', - hidden: true, - }, - 'mock-open-page': { - choices: ['request', 'skip'], - hidden: true, - }, - ...timesOptionsConfig, - }) - .showHidden('show-hidden'); -} - -/** - * @param {AriaATCIAgent.CliOptions} options - * @returns {string[]} - */ -export function agentCliArgsFromOptionsMap(options) { - const args = []; - for (const key of Object.keys(options)) { - const value = options[key]; - switch (key) { - case 'debug': - if (value) { - args.push('--debug'); - } else if (value === false) { - args.push('--debug=false'); - } - break; - case 'quiet': - if (value) { - args.push('--quiet'); - } else if (value === false) { - args.push('--quiet=false'); - } - break; - case 'verbose': - args.push('--verbose', value.join(',')); - break; - case 'referenceBaseUrl': - args.push('--reference-base-url', value.toString()); - break; - case 'webDriverUrl': - args.push('--web-driver-url', value.toString()); - break; - case 'webDriverBrowser': - args.push('--web-driver-browser', value.toString()); - break; - case 'atDriverUrl': - args.push('--at-driver-url', value.toString()); - break; - case 'mock': - if (value) { - args.push('--mock'); - } else if (value === false) { - args.push('--mock=false'); - } - break; - case 'mockOpenPage': - args.push(`--mock-open-page=${value}`); - break; - case 'timesOption': - args.push(...timesArgs(value)); - break; - default: - throw new Error(`unknown agent cli argument ${key}`); - } - } - return args; -} - -/** - * @param {AriaATCIAgent.CliOptions} options - * @returns {AriaATCIAgent.CliOptions} - */ -export function pickAgentCliOptions({ - debug, - quiet, - verbose, - referenceBaseUrl, - webDriverUrl, - webDriverBrowser, - atDriverUrl, - mock, - mockOpenPage, - timesOption, -}) { - return { - ...(debug === undefined ? {} : { debug }), - ...(quiet === undefined ? {} : { quiet }), - ...(verbose === undefined ? {} : { verbose }), - ...(referenceBaseUrl === undefined ? {} : { referenceBaseUrl }), - ...(webDriverUrl === undefined ? {} : { webDriverUrl }), - ...(webDriverBrowser === undefined ? {} : { webDriverBrowser }), - ...(atDriverUrl === undefined ? {} : { atDriverUrl }), - ...(mock === undefined ? {} : { mock }), - ...(mockOpenPage === undefined ? {} : { mockOpenPage }), - timesOption, - }; -} - -/** - * @param {object} options - * @param {import("events").EventEmitter} options.signals - * @param {function(*): void} [options.send] - * @param {import("events").EventEmitter} options.stdin - * @param {import("events").EventEmitter} options.stdout - * @param {import("events").EventEmitter} options.stderr - */ -export async function createAgentCliParser({ signals, send, stdin, stdout, stderr }) { - return /** @type {yargs} */ (await yargs()) - .middleware(argv => { - argv.signals = signals; - argv.send = send; - argv.stdin = stdin; - argv.stdout = stdout; - argv.stderr = stderr; - }) - .command('$0', 'Run tests from input', buildAgentCliOptions, stopAfterMain, [ - agentVerboseMiddleware, - agentAbortMiddleware, - agentLoggerMiddleware, - agentTestsMiddleware, - agentReportMiddleware, - agentRunnerMiddleware, - ]); -} - -/** - * @param {object} options - * @param {object} options.argv - * @param {import("events").EventEmitter} options.signals - * @param {function(*): void} [options.send] - * @param {import("events").EventEmitter} options.stdin - * @param {import("events").EventEmitter} options.stdout - * @param {import("events").EventEmitter} options.stderr - */ -export async function parseAgentCli({ argv, ...parserConfiguration }) { - return await (await createAgentCliParser(parserConfiguration)).parse(hideBin(argv)); -} - -/** - * Summarize cli options as mock options for creating a test runner. - * @param {AriaATCIAgent.CliOptions} cliOptions - * @returns {AriaATCIAgent.MockOptions} - */ -export function agentMockOptions(cliOptions) { - let { mock, mockOpenPage } = pickAgentCliOptions(cliOptions); - if (mock === undefined && mockOpenPage) { - mock = true; - } - if (mock) { - return { openPage: mockOpenPage ? mockOpenPage : 'request' }; - } - return undefined; -} - -async function stopAfterMain(argv) { - await agentMain(argv); - await argv.stop(); -} - -/** - * Build and assign a test runner based on passed arguments. - * @param {object} argv - * @param {AriaATCIAgent.Log} argv.log - * @param {AriaATCIShared.BaseURL} argv.referenceBaseUrl - * @param {boolean} [argv.mock] - * @param {AriaATCIAgent.TestRunner} argv.runner - * @param {AriaATCIAgent.Browser} [argv.webDriverBrowser] - * @param {AriaATCIShared.BaseURL} argv.webDriverUrl - * @param {AriaATCIShared.BaseURL} argv.atDriverUrl - * @param {Promise} argv.abortSignal - */ -async function agentRunnerMiddleware(argv) { - argv.runner = await createRunner({ - log: argv.log, - baseUrl: argv.referenceBaseUrl, - mock: agentMockOptions(argv), - webDriverUrl: argv.webDriverUrl, - webDriverBrowser: argv.webDriverBrowser, - atDriverUrl: argv.atDriverUrl, - abortSignal: argv.abortSignal, - timesOption: getTimesOption(argv), - }); -} - -/** - * Build and assign a test runner based on passed arguments. - * @param {object} argv - * @param {boolean} argv.debug - * @param {boolean} argv.quiet - * @param {string[]} argv.verbose - * @param {string[]} argv.verbosity - */ -async function agentVerboseMiddleware(argv) { - if (argv.debug) { - argv.verbosity = Object.values(AgentMessage); - } else if (argv.quiet) { - argv.verbosity = []; - } else { - argv.verbosity = - argv.verbose && argv.verbose.length - ? argv.verbose - : [AgentMessage.START, AgentMessage.UNCAUGHT_ERROR, AgentMessage.WILL_STOP]; - } -} - -async function agentAbortMiddleware(argv) { - argv.abortSignal = new Promise(resolve => { - argv.stop = resolve; - argv.signals.once('SIGINT', () => resolve()); - process.once('beforeExit', () => resolve()); - }); -} - -/** - * Build and assign main loop arguments based on passed protocol and other arguments. - * @param {object} argv - * @param {function(*): void} argv.send - * @param {AriaATCIAgent.Log} argv.log - * @param {string[]} argv.verbosity - */ -export function agentLoggerMiddleware(argv) { - if (typeof argv.send !== 'function') { - throw new Error( - `Currently, this command may only be used when launched by a nodejs child_process.fork call.` - ); - } - - const logger = createAgentLogger(); - argv.log = logger.log; - - const { send, verbosity } = argv; - logger.emitter.on('message', message => { - if (verbosity.includes(message.data.type)) { - send({ - type: 'log', - data: message, - }); - } - }); -} - -/** - * Build and assign main loop arguments based on passed protocol and other arguments. - * @param {object} argv - * @param {import("events").EventEmitter} argv.signals - * @param {AriaATCIAgent.TestIterable} argv.tests - */ -export function agentTestsMiddleware(argv) { - const { signals } = argv; - argv.tests = (async function* () { - for await (const message of iterateEmitter(signals, 'message', 'SIGINT')) { - if (message.type === 'task') { - yield message.data; - } else if (message.type === 'stop') { - break; - } - } - })(); -} - -/** - * Build and assign main loop arguments based on passed protocol and other arguments. - * @param {object} argv - * @param {function(*): void} argv.send - * @param {AriaATCIAgent.ReportResult} argv.reportResult - */ -export function agentReportMiddleware(argv) { - if (typeof argv.send !== 'function') { - throw new Error( - `Currently, this command may only be used when launched by a nodejs child_process.fork call.` - ); - } - - const { send } = argv; - argv.reportResult = async function (result) { - send({ type: 'result', data: result }); - }; -} diff --git a/src/agent/index.js b/src/agent/index.js deleted file mode 100644 index 32145dc..0000000 --- a/src/agent/index.js +++ /dev/null @@ -1,17 +0,0 @@ -/** - * @module agent - */ - -import { parseAgentCli } from './cli.js'; - -export async function runCli(argv = process.argv) { - const { stdin, stdout, stderr } = process; - return await parseAgentCli({ - argv, - signals: process, - send: process.send ? process.send.bind(process) : null, - stdin, - stdout, - stderr, - }); -} diff --git a/src/agent/main.js b/src/agent/main.js deleted file mode 100644 index 2ff3085..0000000 --- a/src/agent/main.js +++ /dev/null @@ -1,30 +0,0 @@ -/// - -/** - * @module agent - */ - -import { AgentMessage } from './messages.js'; - -/** - * Run agent's main application loop, waiting for tests, running them, and reporting results. - * @param {object} args - * @param {AriaATCIAgent.TestRunner} args.runner - * @param {AriaATCIAgent.Log} args.log - * @param {AriaATCIAgent.TestIterable} args.tests - * @param {AriaATCIAgent.ReportResult} args.reportResult - */ -export async function agentMain({ runner, log, tests, reportResult }) { - try { - log(AgentMessage.START); - // Wait for a test. - for await (const test of tests) { - // Perform the test and report result. - await reportResult(await runner.run(test)); - } - } catch (error) { - log(AgentMessage.UNCAUGHT_ERROR, { error }); - } finally { - log(AgentMessage.WILL_STOP); - } -} diff --git a/src/agent/messages.js b/src/agent/messages.js deleted file mode 100644 index ad8a417..0000000 --- a/src/agent/messages.js +++ /dev/null @@ -1,55 +0,0 @@ -/// -/// - -/** - * @module agent - */ - -import { createSharedLogger } from '../shared/messages.js'; - -/** @enum {AriaATCIAgent.Message} */ -export const AgentMessage = { - /** @type {'start'} */ - START: 'start', - /** @type {'uncaughtError'} */ - UNCAUGHT_ERROR: 'uncaughtError', - /** @type {'willStop'} */ - WILL_STOP: 'willStop', - /** @type {'startTest'} */ - START_TEST: 'startTest', - /** @type {'openPage'} */ - OPEN_PAGE: 'openPage', - /** @type {'invalidKeys'} */ - INVALID_KEYS: 'invalidKeys', - /** @type {'pressKeys'} */ - PRESS_KEYS: 'pressKeys', - /** @type {'speechEvent'} */ - SPEECH_EVENT: 'speechEvent', - /** @type {'noRunTestSetup'} */ - NO_RUN_TEST_SETUP: 'noRunTestSetup', - /** @type {'atDriverComms'} */ - AT_DRIVER_COMMS: 'atDriverComms', - /** @type {'capabilities'} */ - CAPABILITIES: 'capabilities', -}; - -export const AGENT_TEMPLATES = { - [AgentMessage.START]: () => `Starting...`, - [AgentMessage.UNCAUGHT_ERROR]: ({ error }) => `Uncaught error: ${error.stack || error.message}`, - [AgentMessage.WILL_STOP]: () => `Stopping...`, - [AgentMessage.START_TEST]: ({ id, title }) => `Starting test #${id} '${title}'.`, - [AgentMessage.OPEN_PAGE]: ({ url }) => `Open page: '${url}'.`, - [AgentMessage.INVALID_KEYS]: ({ command, errors }) => - `Keys in '${command.id}' have issues:\n${errors.map(error => `- ${error}`).join('\n')}`, - [AgentMessage.PRESS_KEYS]: ({ keys }) => `Press keys: '${keys.toString()}'.`, - [AgentMessage.SPEECH_EVENT]: ({ spokenText }) => `Speech event: '${spokenText}'.`, - [AgentMessage.NO_RUN_TEST_SETUP]: ({ referencePage }) => - `Test reference, ${referencePage}, does not have a Run Test Setup button.`, - [AgentMessage.AT_DRIVER_COMMS]: ({ direction, message }) => `AT-Driver: ${direction}: ${message}`, - [AgentMessage.CAPABILITIES]: ({ capabilities }) => - `Capabilities: ${JSON.stringify(capabilities)}`, -}; - -export function createAgentLogger(messages = AGENT_TEMPLATES) { - return createSharedLogger(messages); -} diff --git a/src/host/README.md b/src/host/README.md index 868787f..d80e2ff 100644 --- a/src/host/README.md +++ b/src/host/README.md @@ -2,7 +2,7 @@ This directory implements the systems used by the `bin/host.js` command line tool. -The `bin/host.js` tool has two commands. The `run-plan` command operates a server and `bin/agent.js` instance to run test plans and the `read-plan` command reads files from disk and packages them for the `run-plan` command. +The `bin/host.js` tool has two commands. The `run-plan` command operates a server to run test plans and the `read-plan` command reads files from disk and packages them for the `run-plan` command. A short invocation of the `run-plan` command can be made with a list of files or file globs that make up a test plan. A test plan should contain tests for a single assistive technology to be tested. If for example all the reference files and test files are in two directories this command could be executed. @@ -15,10 +15,10 @@ Removing reference from 'http://localhost:52147/gtmoyv'. Stopping... ``` -Developing `bin/host.js` you may want to use the mock test runner in `bin/agent.js`. Calling `host.js` with `--agent-mock` will enable the mock test runner. +Developing `bin/host.js` you may want to use the mock test runner. Calling `host.js` with `--runner-mock` will enable the mock test runner. ```sh -$ bin/host.js run-plan --agent-mock reference/** at/** >result.json +$ bin/host.js run-plan --runner-mock reference/** at/** >result.json ... ``` @@ -26,54 +26,17 @@ The mock test runner will be removed in the near future and replaced with the ab ## "main" command -The `host` loads test plans, serves their files from a http server, runs each test through an `agent` instance, and reports the collected results for each test plan. +The `host` loads test plans, serves their files from a http server, runs each test, and reports the collected results for each test plan. 1. Start a server listening on a port 1. Read a test plan from disk or input stream 1. Add test plan files to a subdirectory on the server -1. Start an `agent` instance -1. Run each test in the test plan through the `agent` instance -1. Stop the `agent` instance +1. Run each test in the test plan 1. Emit the results of the test in a json format 1. If reading plans from stream, repeat from step 2 1. Stop the server 1. Gracefully exit -``` -$ bin/host.js run-plan --help --show-hidden -host.js run-plan [plan-files..] - -Run test plans - -Positionals: - plan-files Files in a test plan [array] [default: []] - -Options: - --help Show help [boolean] - --version Show version number [boolean] - --quiet Disable all logging - --debug Enable all logging - --verbose Enable a subset of logging messages - --tests-match Files matching pattern in a test plan will be test - ed [string] [default: "{,**/}test*.json"] - --reference-hostname [default: "localhost"] - --plan-workingdir Directory "plan-files" are relative to - [default: "."] - --plan-protocol [choices: "fork", "developer"] [default: "fork"] - --agent-web-driver-url [default: "http://localhost:4444"] - --agent-web-driver-browser [choices: "chrome", "firefox"] [default: "firefox"] - --agent-at-driver-url [default: "http://localhost:4382"] - --agent-protocol [choices: "fork", "developer"] [default: "fork"] - --agent-quiet Disable all logging - --agent-debug Enable all logging - --agent-verbose Enable a subset of logging messages - --agent-mock [boolean] - --agent-mock-open-page [choices: "request", "skip"] - --callback-url URL to POST test results to as they complete - --callback-header Header to send with callback request - --show-hidden Show hidden options [boolean] -``` - ### Loading a test plan `bin/host.js` can load a test plan through the `read-plan` command (or developer @@ -81,10 +44,6 @@ interface) with `src/host/plan-from.js`. For convenience the `run-plan` command takes arguments prefixed with `plan` that map to arguments that can be passed to `read-plan`, to read a plan. -### `bin/agent.js` communication - -The host while managing an agent instance can operate it through the `bin/agent.js` tool or `src/agent/main.js` developer interface. - ### `--verbose` options The main command's verbose level can be set with `--debug`, `--quiet`, or `--verbose`. `--verbose` takes a comma separate list of the following logging message types. @@ -116,5 +75,3 @@ Options: --version Show version number [boolean] --workingdir Directory to read files from [string] [default: "."] ``` - -Currently this command must be executed by a parent node process as a node fork child process. diff --git a/src/host/agent.js b/src/host/agent.js deleted file mode 100644 index 82ec12d..0000000 --- a/src/host/agent.js +++ /dev/null @@ -1,354 +0,0 @@ -/// -/// -/// - -/** - * @module host - */ - -import * as child_process from 'child_process'; -import { EventEmitter } from 'events'; -import { constants as osConstants } from 'os'; -import * as path from 'path'; -import { fileURLToPath, URL } from 'url'; - -import { iterateEmitter } from '../shared/iterate-emitter.js'; -import { processExited, collectProcessPipe } from '../shared/process-util.js'; - -import { agentCliArgsFromOptionsMap, agentMockOptions } from '../agent/cli.js'; -import { createRunner } from '../agent/create-test-runner.js'; -import { agentMain } from '../agent/main.js'; -import { AgentMessage, createAgentLogger } from '../agent/messages.js'; - -import { HostMessage } from './messages.js'; - -const { - signals: { SIGINT }, -} = osConstants; - -export class AgentController { - /** - * @param {object} options - * @param {AriaATCIHost.Log} options.log - * @param {'fork' | 'developer'} [options.protocol] - * @param {AriaATCIAgent.CliOptions} [options.config] - */ - constructor({ config = {}, ...otherOptions }) { - this._options = { - ...this._defaultOptions(), - ...otherOptions, - config: this._modifyConfig(config), - }; - - /** @type {AriaATCIAgent.CliOptions} */ - this._activeConfig = null; - /** @type {AgentProtocol} */ - this._activeProtocol = null; - - this._logEmitter = new EventEmitter(); - } - - _defaultOptions() { - return { protocol: 'fork' }; - } - - /** - * @param {AriaATCIAgent.CliOptions} config - * @returns {AriaATCIAgent.CliOptions} - */ - _modifyConfig(config) { - return { - debug: config.quiet === undefined && config.verbose === undefined ? true : undefined, - ...config, - }; - } - - /** - * @param {AriaATCIData.Test} test - * @returns {Promise} - */ - async run(test) { - return await this._activeProtocol.run(test); - } - - /** - * @returns {AsyncGenerator} - */ - async *logs() { - if (this._activeProtocol) { - yield* this._activeProtocol.logs(); - } - for await (const instance of iterateEmitter(this._logEmitter, 'log')) { - yield* instance.logs(); - } - } - - /** - * @param {AriaATCIAgent.CliOptions} [cliOptions] - */ - async start(cliOptions) { - try { - const { log, protocol, config } = this._options; - const mergedOptions = { ...config, ...cliOptions }; - this._activeConfig = mergedOptions; - - const Protocol = AGENT_PROTOCOL[protocol]; - this._activeProtocol = new Protocol(); - const ready = this._activeProtocol.start(mergedOptions); - this._logEmitter.emit('log', this._activeProtocol); - await ready; - log(HostMessage.AGENT_PROTOCOL, { protocol: Protocol.protocolName }); - } catch (error) { - this._activeProtocol = null; - throw new Error(`Agent failed to start.\n${error.stack ? error.stack : error.toString()}`); - } - } - - async stop() { - await this._activeProtocol.stop(); - this._activeProtocol = null; - } -} - -class AgentProtocol { - static get protocolName() { - return 'unknown'; - } - - /** - * Start a copy of src/agent and run tests in it. - */ - constructor() { - /** @type {Promise<{code: number, signal: number}> | null} */ - this.exited = null; - /** @type {Promise | null} */ - this.ready = null; - } - - /** - * @param {AriaATCIData.Test} test - * @returns {Promise} - */ - async run(test) { - const { testId } = test.info; - this._sendTest(test); - for await (const result of this._results()) { - if (result.testId === testId) { - return result; - } - } - throw new Error('test result not received'); - } - - /** - * Iterate process log messages as they are received. - * @returns {AsyncGenerator} - */ - async *logs() { - throw new Error(`${this.constructor.name}.logs() not implemented`); - } - - /** - * Start the agent process. - * @param {AriaATCIAgent.CliOptions} options - */ - async start(options) { - throw new Error(`${this.constructor.name}.start() not implemented`); - } - - /** - * Stop the agent process. - */ - async stop() { - throw new Error(`${this.constructor.name}.stop() not implemented`); - } - - /** - * Send a test to the process. - * @param {AriaATCIData.Test} test - */ - async _sendTest(test) { - throw new Error(`${this.constructor.name}._sendTest() not implemented`); - } - - /** - * Iterate results from the agent process as they are received. - * @returns {AsyncGenerator} - */ - async *_results() { - throw new Error(`${this.constructor.name}._results() not implemented`); - } -} - -class AgentForkProtocol extends AgentProtocol { - static get protocolName() { - return 'fork'; - } - - /** - * Start a copy of src/agent with child_process.fork and run tests in it. - */ - constructor() { - super(); - /** @type {child_process.ChildProcess | null} */ - this._processFork = null; - } - - async *logs() { - for await (const message of this._messages()) { - if (message.type === 'log') { - yield message.data; - } - } - } - - /** - * @param {AriaATCIAgent.CliOptions} options - */ - async start(options) { - const agentPath = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - '../../bin/agent.js' - ); - - const agentProcess = (this._processFork = child_process.fork( - agentPath, - agentCliArgsFromOptionsMap(options), - { stdio: 'pipe', serialization: 'advanced' } - )); - - const stderrJob = collectProcessPipe(agentProcess.stderr); - this.exited = processExited(agentProcess); - this.ready = (async () => { - for await (const log of this.logs()) { - if (log.data.type === AgentMessage.START) { - await stderrJob.cancel(); - return; - } - } - const stderrOutput = await stderrJob.cancel(); - throw new Error(`Agent fork exited before it was ready.\n${stderrOutput}`); - })(); - - await this.ready; - } - - async stop() { - if (this._processFork) { - this._processFork.send({ type: 'stop' }); - try { - await Promise.race([ - this.exited, - new Promise(resolve => setTimeout(resolve, 1000)).then(() => { - throw new Error('AgentProtocol: graceful stop timeout'); - }), - ]); - } catch (_) { - this._processFork.kill(SIGINT); - } - } - await this.exited; - - this._processFork = null; - this.exited = null; - this.ready = null; - } - - async _sendTest(test) { - this._processFork.send({ type: 'task', data: test }); - } - - async *_messages() { - yield* iterateEmitter(this._processFork, 'message', 'exit'); - } - - async *_results() { - for await (const message of this._messages()) { - if (message.type === 'result') { - yield message.data; - } - } - } -} - -class AgentDeveloperProtocol extends AgentProtocol { - static get protocolName() { - return 'developer'; - } - - /** - * Start a copy of src/agent with the js api and run tests in it. - */ - constructor() { - super(); - this._testEmitter = null; - this._logEmitter = null; - this._resultEmitter = null; - } - - async _sendTest(test) { - this._testEmitter.emit('message', test); - } - - _results() { - return iterateEmitter(this._resultEmitter, 'result', 'stop'); - } - - async *logs() { - yield* iterateEmitter(this._logEmitter, 'message', 'stop'); - } - - /** - * @param {AriaATCIAgent.CliOptions} options - */ - async start(options) { - const { log, emitter: logEmitter } = createAgentLogger(); - this._testEmitter = new EventEmitter(); - this._logEmitter = logEmitter; - this._resultEmitter = new EventEmitter(); - - const abortSignal = new Promise(resolve => { - this._testEmitter.once('stop', () => resolve()); - }); - this.exited = agentMain({ - runner: await createRunner({ - abortSignal, - baseUrl: options.referenceBaseUrl || new URL('http://localhost:4400'), - log, - mock: agentMockOptions(options), - atDriverUrl: options.atDriverUrl, - webDriverBrowser: options.webDriverBrowser, - webDriverUrl: options.webDriverUrl, - timesOption: options.timesOption, - }), - log, - tests: iterateEmitter(this._testEmitter, 'message', 'stop'), - reportResult: async result => { - this._resultEmitter.emit('result', result); - }, - }) - .then(() => { - this.stop(); - }) - .then(() => ({ code: 0, signal: null })); - this.ready = Promise.resolve(); - - await this.ready; - } - - async stop() { - this._testEmitter.emit('stop'); - this._logEmitter.emit('stop'); - this._resultEmitter.emit('stop'); - - await this.exited; - - this.exited = null; - this.ready = null; - } -} - -const AGENT_PROTOCOL = { - fork: AgentForkProtocol, - developer: AgentDeveloperProtocol, -}; diff --git a/src/host/cli-run-plan.js b/src/host/cli-run-plan.js index 9e26a1a..095f65b 100644 --- a/src/host/cli-run-plan.js +++ b/src/host/cli-run-plan.js @@ -7,15 +7,13 @@ import { Readable } from 'stream'; import fetch, { Response } from 'node-fetch'; import yargs from 'yargs'; -import { pickAgentCliOptions } from '../agent/cli.js'; -import { AgentMessage } from '../agent/messages.js'; +import { RunnerMessage } from '../runner/messages.js'; -import { AgentController as Agent } from './agent.js'; import { hostMain } from './main.js'; import { HostMessage, createHostLogger } from './messages.js'; import { plansFrom } from './plan-from.js'; import { HostServer } from './server.js'; -import { getTimesOption, timesOptionsConfig } from '../shared/times-option.js'; +import { timesOptionsConfig } from '../shared/times-option.js'; export const command = 'run-plan [plan-files..]'; @@ -77,68 +75,26 @@ export const builder = (args = yargs) => default: 'fork', hidden: true, }, - 'agent-web-driver-url': { + 'web-driver-url': { coerce(arg) { return new URL(arg); }, default: 'http://localhost:4444', }, - 'agent-web-driver-browser': { + 'web-driver-browser': { choices: ['chrome', 'firefox', 'safari'], default: 'firefox', }, - 'agent-at-driver-url': { + 'at-driver-url': { coerce(arg) { return new URL(arg); }, default: 'http://localhost:4382', }, - 'agent-protocol': { - choices: ['fork', 'developer'], - default: 'fork', - hidden: true, - }, - 'agent-quiet': { - conflicts: ['agent-debug', 'agent-verbose'], - describe: 'Disable all logging', - hidden: true, - }, - 'agent-debug': { - conflicts: ['agent-quiet', 'agent-verbose'], - describe: 'Enable all logging', - hidden: true, - }, - 'agent-verbose': { - coerce(arg) { - if (!arg) { - return; - } - const messageValues = Object.values(AgentMessage); - const verbosity = arg.split(','); - for (const name of verbosity) { - if (!messageValues.includes(name)) { - throw new Error( - `--verbose must be a comma separated list including: ${Object.values( - AgentMessage - ).join(', ')}` - ); - } - } - return verbosity; - }, - conflicts: ['agent-debug', 'agent-quiet'], - describe: 'Enable a subset of logging messages', - nargs: 1, - hidden: true, - }, - 'agent-mock': { + 'runner-mock': { type: 'boolean', hidden: true, }, - 'agent-mock-open-page': { - choices: ['request', 'skip'], - hidden: true, - }, 'callback-url': { describe: 'URL to POST test results to as they complete', }, @@ -170,7 +126,7 @@ async function verboseMiddleware(argv) { let verbosity; if (debug) { - verbosity = Object.values(HostMessage); + verbosity = Object.values({ ...HostMessage, ...RunnerMessage }); } else if (quiet) { verbosity = []; } else { @@ -183,6 +139,8 @@ async function verboseMiddleware(argv) { HostMessage.SERVER_LISTENING, HostMessage.ADD_SERVER_DIRECTORY, HostMessage.REMOVE_SERVER_DIRECTORY, + HostMessage.UNCAUGHT_ERROR, + RunnerMessage.OPEN_PAGE, ]; } @@ -195,13 +153,12 @@ function mainMiddleware(argv) { mainLoggerMiddleware(argv); mainTestPlanMiddleware(argv); mainServerMiddleware(argv); - mainAgentMiddleware(argv); mainResultMiddleware(argv); } function mainFetchMiddleware(argv) { if (!argv.fetch) { - if (!argv.agentMock) { + if (!argv.runnerMock) { argv.fetch = fetch; } else { argv.fetch = (url, ...params) => @@ -228,6 +185,7 @@ function mainLoggerMiddleware(argv) { const logger = createHostLogger(); argv.log = logger.log; + argv.logger = logger; logger.emitter.on('message', ({ data: { type }, text }) => { if (verbosity.includes(type)) { @@ -260,39 +218,6 @@ function mainServerMiddleware(argv) { argv.server = new HostServer({ log, baseUrl: { hostname: argv.referenceHostname } }); } -function mainAgentMiddleware(argv) { - const { - log, - agentProtocol: protocol, - agentDebug, - agentQuiet, - agentVerbose, - agentWebDriverUrl, - agentWebDriverBrowser, - agentAtDriverUrl, - agentMock, - agentMockOpenPage, - } = argv; - - const timesOption = getTimesOption(argv); - - argv.agent = new Agent({ - log, - protocol, - config: pickAgentCliOptions({ - debug: agentDebug, - quiet: agentQuiet, - verbose: agentVerbose, - webDriverUrl: agentWebDriverUrl, - webDriverBrowser: agentWebDriverBrowser, - atDriverUrl: agentAtDriverUrl, - mock: agentMock, - mockOpenPage: agentMockOpenPage, - timesOption: timesOption, - }), - }); -} - function mainResultMiddleware(argv) { const { stdout } = argv; diff --git a/src/host/main.js b/src/host/main.js index 7dfb9d9..e541640 100644 --- a/src/host/main.js +++ b/src/host/main.js @@ -4,7 +4,7 @@ * @module host */ -import { startJob } from '../shared/job.js'; +import { createRunner } from '../runner/create-test-runner.js'; import { HostMessage } from './messages.js'; import { @@ -13,6 +13,8 @@ import { addTestLogToTestPlan, addTestResultToTestPlan, } from './plan-object.js'; +import { getTimesOption } from '../shared/times-option.js'; +import { RUNNER_TEMPLATES } from '../runner/messages.js'; /** * @param {AriaATCIHost.Log} log @@ -29,33 +31,35 @@ const logUnsuccessfulHTTP = async (log, response) => { /** * @param {object} options - * @param {AriaATCIHost.Log} options.log + * @param {AriaATCIHost.Logger} options.logger * @param {AsyncIterable} options.plans * @param {AriaATCIHost.ReferenceFileServer} options.server - * @param {AriaATCIHost.Agent} options.agent + * @param {AriaATCIRunner.TestRunner} options.runner * @param {AriaATCIHost.EmitPlanResults} options.emitPlanResults * @param {string} [options.callbackUrl] * @param {Record} [options.callbackHeader] * @param {typeof fetch} options.fetch + * @param {boolean} options.runnerMock + * @param {AriaATCIShared.BaseURL} options.webDriverUrl + * @param {AriaATCIRunner.Browser} options.webDriverBrowser + * @param {AriaATCIShared.BaseURL} options.atDriverUrl */ -export async function hostMain({ - log, - plans, - server, - agent, - emitPlanResults, - callbackUrl, - callbackHeader, - fetch, -}) { +export async function hostMain(options) { + const { + logger, + plans, + server, + emitPlanResults, + callbackUrl, + callbackHeader, + runnerMock, + webDriverUrl, + webDriverBrowser, + atDriverUrl, + } = options; + const { log } = logger; log(HostMessage.START); - const hostLogJob = startJob(async function (signal) { - for await (const agentLog of signal.cancelable(agent.logs())) { - log(HostMessage.AGENT_LOG, agentLog); - } - }); - await server.ready; log(HostMessage.SERVER_LISTENING, { url: server.baseUrl }); @@ -65,8 +69,26 @@ export async function hostMain({ log(HostMessage.ADD_SERVER_DIRECTORY, { url: serverDirectory.baseUrl }); setServerOptionsInTestPlan(plan, { baseUrl: serverDirectory.baseUrl }); - log(HostMessage.START_AGENT); - await agent.start({ referenceBaseUrl: serverDirectory.baseUrl }); + const timesOption = getTimesOption(options); + + let stopDrivers = any => {}; + const abortSignal = new Promise(resolve => { + stopDrivers = () => { + log(HostMessage.STOP_DRIVERS); + resolve(); + }; + }); + + const runner = await createRunner({ + log, + abortSignal, + timesOption, + baseUrl: new URL(serverDirectory.baseUrl.toString()), + mock: runnerMock, + webDriverUrl, + webDriverBrowser, + atDriverUrl, + }); let lastCallbackRequest = Promise.resolve(); @@ -82,23 +104,17 @@ export async function hostMain({ body.presentationNumber ?? body.testCsvRow ); lastCallbackRequest = lastCallbackRequest.then(() => - fetch(perTestUrl, { - method: 'post', - body: JSON.stringify(body), - headers, - }).then(logUnsuccessfulHTTP.bind(null, log)) + options + .fetch(perTestUrl, { + method: 'post', + body: JSON.stringify(body), + headers, + }) + .then(logUnsuccessfulHTTP.bind(null, log)) ); }; for (const test of plan.tests) { - log(HostMessage.START_TEST); - const testLogJob = startJob(async function (signal) { - for await (const testLog of signal.cancelable(agent.logs())) { - plan = addLogToTestPlan(plan, testLog); - plan = addTestLogToTestPlan(plan, test); - } - }); - const file = plan.files.find(({ name }) => name === test.filepath); const testSource = JSON.parse(textDecoder.decode(file.bufferData)); @@ -106,10 +122,19 @@ export async function hostMain({ const callbackBody = presentationNumber ? { presentationNumber } : { testCsvRow }; + log(HostMessage.START_TEST, { id: testSource.info.testId, title: testSource.info.title }); + const addLogtoPlan = message => { + if (Object.keys(RUNNER_TEMPLATES).includes(message.data.type)) { + plan = addLogToTestPlan(plan, message); + plan = addTestLogToTestPlan(plan, test); + } + }; + logger.emitter.on('message', addLogtoPlan); + try { postCallbackWhenEnabled({ ...callbackBody, status: 'RUNNING' }); - const result = await agent.run(testSource); + const result = await runner.run(testSource); const { capabilities, commands } = result; @@ -128,21 +153,20 @@ export async function hostMain({ await lastCallbackRequest; throw exception; } finally { - await testLogJob.cancel(); + logger.emitter.off('message', addLogtoPlan); } } server.removeFiles(serverDirectory); log(HostMessage.REMOVE_SERVER_DIRECTORY, { url: serverDirectory.baseUrl }); - log(HostMessage.STOP_AGENT); await lastCallbackRequest; - await agent.stop(); + + stopDrivers(); + await emitPlanResults(plan); } - await hostLogJob.cancel(); - log(HostMessage.STOP_SERVER); await server.close(); diff --git a/src/host/messages.js b/src/host/messages.js index b8e40af..3baa2fd 100644 --- a/src/host/messages.js +++ b/src/host/messages.js @@ -4,6 +4,7 @@ * @module host */ +import { RUNNER_TEMPLATES } from '../runner/messages.js'; import { createSharedLogger } from '../shared/messages.js'; /** @enum {AriaATCIHost.HostLogType} */ @@ -22,22 +23,14 @@ export const HostMessage = { SERVER_LISTENING: 'serverListening', /** @type {'stopServer'} */ STOP_SERVER: 'stopServer', + /** @type {'stopDrivers'} */ + STOP_DRIVERS: 'stopDrivers', /** @type {'addServerDirectory'} */ ADD_SERVER_DIRECTORY: 'addServerDirectory', /** @type {'removeServerDirectory'} */ REMOVE_SERVER_DIRECTORY: 'removeServerDirectory', /** @type {'serverLog'} */ SERVER_LOG: 'serverLog', - /** @type {'startAgent'} */ - START_AGENT: 'startAgent', - /** @type {'agentProtocol'} */ - AGENT_PROTOCOL: 'agentProtocol', - /** @type {'stopAgent'} */ - STOP_AGENT: 'stopAgent', - /** @type {'agentLog'} */ - AGENT_LOG: 'agentLog', - /** @type {'agentCrashed'} */ - AGENT_CRASHED: 'agentCrashed', /** @type {'startTest'} */ START_TEST: 'startTest', /** @type {'reportingError'} */ @@ -55,14 +48,10 @@ export const HOST_TEMPLATES = { [HostMessage.START_SERVER]: () => `Starting reference server.`, [HostMessage.SERVER_LISTENING]: ({ url }) => `Reference server listening on '${url}'.`, [HostMessage.STOP_SERVER]: () => `Stopping reference server.`, + [HostMessage.STOP_DRIVERS]: () => `Stopping drivers.`, [HostMessage.ADD_SERVER_DIRECTORY]: ({ url }) => `Reference available on '${url}'.`, [HostMessage.REMOVE_SERVER_DIRECTORY]: ({ url }) => `Removing reference from '${url}'.`, [HostMessage.SERVER_LOG]: ({ text }) => `[Server]: ${text}`, - [HostMessage.START_AGENT]: () => `Starting test agent.`, - [HostMessage.AGENT_PROTOCOL]: ({ protocol }) => `Agent running with protocol '${protocol}'.`, - [HostMessage.STOP_AGENT]: () => `Stopping test agent.`, - [HostMessage.AGENT_LOG]: ({ text }) => `[Agent]: ${text}`, - [HostMessage.AGENT_CRASHED]: () => `Agent crashed.`, [HostMessage.START_TEST]: () => `Starting test.`, [HostMessage.TEST_ERROR]: ({ error }) => `Test Error ${error}`, [HostMessage.REPORTING_ERROR]: ({ status, body }) => @@ -73,6 +62,6 @@ export const HOST_TEMPLATES = { * @param {*} messages * @returns {{log: AriaATCIHost.Log, emitter: import("events").EventEmitter}} */ -export function createHostLogger(messages = HOST_TEMPLATES) { +export function createHostLogger(messages = { ...HOST_TEMPLATES, ...RUNNER_TEMPLATES }) { return createSharedLogger(messages); } diff --git a/src/host/tests/agent.js b/src/host/tests/agent.js deleted file mode 100644 index 4c61fdb..0000000 --- a/src/host/tests/agent.js +++ /dev/null @@ -1,171 +0,0 @@ -/// - -import test from 'ava'; - -import { iterateEmitter } from '../../shared/iterate-emitter.js'; -import { startJob } from '../../shared/job.js'; -import { AgentMessage } from '../../agent/messages.js'; - -import { AgentController } from '../agent.js'; -import { createHostLogger, HostMessage } from '../messages.js'; - -test('new AgentController(options)', async t => { - t.timeout(60000); - const TEST_DEFINITIONS = []; - for (const tests of createTests()) { - for (const protocol of /**@type {('fork'|'developer')[]}*/ ([undefined, 'fork', 'developer'])) { - for (const config of [ - {}, - { debug: true }, - { verbose: [AgentMessage.START] }, - { - referenceBaseUrl: { - protocol: 'http:', - hostname: 'localhost', - port: 1234, - pathname: '/path', - toString() { - return `${this.protocol}//${this.hostname}:${this.port}${this.pathname}`; - }, - }, - }, - ]) { - TEST_DEFINITIONS.push({ - tests, - options: { protocol, config }, - }); - } - } - } - - t.log(`${TEST_DEFINITIONS.length} definitions`); - t.is(TEST_DEFINITIONS.length, 36); - - for (const testDefinition of TEST_DEFINITIONS) { - const { log, emitter } = createHostLogger(); - const logJob = startJob(async ({ cancelable }) => { - const logs = []; - for await (const log of cancelable(iterateEmitter(emitter, 'message', 'exit', 'error'))) { - logs.push(omitDates(log)); - } - return logs; - }); - - const controller = new AgentController({ - log, - ...(testDefinition.options.protocol && { - protocol: testDefinition.options.protocol, - }), - config: { - ...testDefinition.options.config, - mock: true, - mockOpenPage: 'skip', - }, - }); - - const agentLogJob = startJob(async ({ cancelable }) => { - for await (const message of cancelable(controller.logs())) { - log(HostMessage.AGENT_LOG, message); - } - }); - - await controller.start(); - - for (const test of testDefinition.tests) { - t.snapshot( - await controller.run(test), - `${snapshotPrefix(testDefinition)}: controller.run(${test.info.title})` - ); - } - - await controller.stop(); - await agentLogJob.cancel(); - const fullLog = await logJob.cancel(); - t.snapshot(fullLog, `${snapshotPrefix(testDefinition)}: log`); - } -}); - -/** - * @returns {AriaATCIData.CollectedTest[][]} - */ -function createTests() { - return [ - [], - [ - { - info: { testId: 1, title: 'test 1', task: 'test', references: [] }, - target: { - at: { key: 'at', name: 'At', raw: 'AT' }, - mode: 'reading', - referencePage: 'reference/index.html', - }, - instructions: { raw: '', user: [] }, - commands: [ - { - id: 'UP_ARROW', - keystroke: 'up arrow', - keypresses: [{ id: 'UP_ARROW', keystroke: 'up arrow' }], - }, - ], - assertions: [{ expectation: 'role up', priority: 1 }], - }, - ], - [ - { - info: { testId: 1, title: 'test 1', task: 'test', references: [] }, - target: { - at: { key: 'at', name: 'At', raw: 'AT' }, - mode: 'reading', - referencePage: 'reference/index.html', - }, - instructions: { raw: '', user: [] }, - commands: [ - { - id: 'UP_ARROW', - keystroke: 'up arrow', - keypresses: [{ id: 'UP_ARROW', keystroke: 'up arrow' }], - }, - ], - assertions: [{ expectation: 'role up', priority: 1 }], - }, - { - info: { testId: 2, title: 'test 2', task: 'interaction', references: [] }, - target: { - at: { key: 'at', name: 'At', raw: 'AT' }, - mode: 'interaction', - referencePage: 'reference/index.html', - }, - instructions: { raw: '', user: [] }, - commands: [ - { - id: 'UP_ARROW,DOWN_ARROW', - keystroke: 'up arrow, then down arrow', - keypresses: [ - { id: 'UP_ARROW', keystroke: 'up arrow' }, - { id: 'DOWN_ARROW', keystroke: 'down arrow' }, - ], - }, - ], - assertions: [{ expectation: 'role down', priority: 1 }], - }, - ], - ]; -} - -function snapshotPrefix({ tests, options }) { - return JSON.stringify({ tests: tests.map(test => test.info.testId), options }); -} - -function omitDates(obj) { - if (Array.isArray(obj)) { - return obj.map(omitDates); - } - if (typeof obj === 'object' && obj !== null) { - return Object.fromEntries( - Object.entries(obj) - .map(([key, value]) => (key === 'date' ? null : [key, omitDates(value)])) - .filter(Boolean) - ); - } - return obj; -} diff --git a/src/host/tests/cli-run-plan.js b/src/host/tests/cli-run-plan.js index 3a870f3..01b8e30 100644 --- a/src/host/tests/cli-run-plan.js +++ b/src/host/tests/cli-run-plan.js @@ -3,14 +3,13 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import test from 'ava'; -import { HostMessage } from '../messages.js'; test('plan1', async t => { t.snapshot( await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan1', '"**"', - '--agent-mock', + '--runner-mock', '--debug', ]) ); @@ -21,7 +20,7 @@ test('plan2', async t => { await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan2', '"**"', - '--agent-mock', + '--runner-mock', '--debug', ]) ); @@ -32,7 +31,7 @@ test('plan3', async t => { await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', '"**"', - '--agent-mock', + '--runner-mock', '--debug', ]) ); @@ -43,7 +42,7 @@ test('plan3 with callback no url param', async t => { await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', '"**"', - '--agent-mock', + '--runner-mock', '--debug', '--callback-url=http://callback.url/', '--callback-header=test:header:multiple:colon', @@ -56,7 +55,7 @@ test('plan3 with callback', async t => { await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', '"**"', - '--agent-mock', + '--runner-mock', '--debug', '--callback-url=http://callback.url/:testRowNumber', '--callback-header=test:header:multiple:colon', @@ -69,7 +68,7 @@ test('plan3 with callback request which fails', async t => { await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', '"**"', - '--agent-mock', + '--runner-mock', '--debug', '--callback-url=http://example.com/?TEST-STATUS=418', '--callback-header=x:y', @@ -82,7 +81,7 @@ test('plan3 with callback request which fails with a faulty response body', asyn await spawnRunPlan([ '--plan-workingdir=fixtures/host-bin/plan3', '"**"', - '--agent-mock', + '--runner-mock', '--debug', '--callback-url="http://example.com/?TEST-STATUS=418&TEST-BAD-BODY"', '--callback-header=x:y', diff --git a/src/host/tests/messages.js b/src/host/tests/messages.js index 0b4c4e7..6067994 100644 --- a/src/host/tests/messages.js +++ b/src/host/tests/messages.js @@ -3,7 +3,7 @@ import test from 'ava'; import { HostMessage, createHostLogger } from '../messages.js'; test('log', async t => { - t.plan(16); + t.plan(11); const { log, emitter } = createHostLogger(); const logAndResolveMessage = async (type, more) => { const message = await new Promise(resolve => { @@ -59,10 +59,5 @@ test('log', async t => { t.snapshot( await logAndResolveMessage(HostMessage.SERVER_LOG, { text: `Served file 'test.json'.` }) ); - t.snapshot(await logAndResolveMessage(HostMessage.START_AGENT)); - t.snapshot(await logAndResolveMessage(HostMessage.AGENT_PROTOCOL, { protocol: 'fork' })); - t.snapshot(await logAndResolveMessage(HostMessage.STOP_AGENT)); - t.snapshot(await logAndResolveMessage(HostMessage.AGENT_LOG, { text: 'Starting test.' })); - t.snapshot(await logAndResolveMessage(HostMessage.AGENT_CRASHED)); t.snapshot(await logAndResolveMessage(HostMessage.START_TEST)); }); diff --git a/src/host/tests/snapshots/cli-run-plan.js.md b/src/host/tests/snapshots/cli-run-plan.js.md index 86dccb8..0ac34f4 100644 --- a/src/host/tests/snapshots/cli-run-plan.js.md +++ b/src/host/tests/snapshots/cli-run-plan.js.md @@ -13,22 +13,18 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 3 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ [Server]: Serving '/static/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/index.html'.␊ + Starting test #2 'test 2'.␊ [Server]: Serving '/static/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/index.html'.␊ + Open page: 'http://localhost:8888/static/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/index.html"},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/index.html"},"text":"Open page: 'http://localhost:8888/static/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/index.html"},"text":"Open page: 'http://localhost:8888/static/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/index.html"},"text":"Open page: 'http://localhost:8888/static/index.html'."}]}␊ `, } @@ -41,22 +37,18 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -69,22 +61,18 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, - stdout: `{"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + stdout: `{"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -97,18 +85,14 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, @@ -132,7 +116,7 @@ Generated by [AVA](https://avajs.dev). body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -145,18 +129,14 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, @@ -180,7 +160,7 @@ Generated by [AVA](https://avajs.dev). body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', test: 'header:multiple:colon' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -193,22 +173,18 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ HTTP 418 response received when reporting result: 'a body'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ HTTP 418 response received when reporting result: 'a body'.␊ HTTP 418 response received when reporting result: 'a body'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ HTTP 418 response received when reporting result: 'a body'.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, @@ -232,7 +208,7 @@ Generated by [AVA](https://avajs.dev). body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } @@ -245,22 +221,18 @@ Generated by [AVA](https://avajs.dev). Reference server listening on 'http://localhost:8888'.␊ Plan 'unknown' with 2 tests and 4 files read from source 'fork'.␊ Reference available on 'http://localhost:8888/static␊ - Starting test agent.␊ - [Agent]: Starting...␊ - Agent running with protocol 'fork'.␊ - Starting test.␊ + Starting test #1 'test 1'.␊ HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ - Starting test.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Starting test #2 'test 2'.␊ HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ [Server]: Serving '/static/reference/index.html'.␊ - [Agent]: Open page: 'http://localhost:8888/static/reference/index.html'.␊ + Open page: 'http://localhost:8888/static/reference/index.html'.␊ Removing reference from 'http://localhost:8888/static␊ - Stopping test agent.␊ HTTP 418 response received when reporting result: 'Unknown error - unable to read response body.'.␊ - [Agent]: Stopping...␊ + Stopping drivers.␊ Stopping reference server.␊ Stopping...␊ `, @@ -284,6 +256,6 @@ Generated by [AVA](https://avajs.dev). body: '{"testCsvRow":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"status":"COMPLETED","responses":["mocked output for UP_ARROW,DOWN_ARROW"]}',␊ headers: { 'Content-Type': 'application/json', x: 'y' }␊ }␊ - {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":{}},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ + {"name":"unknown","tests":[{"filepath":"tests/test-1.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":1,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW","output":"mocked output for UP_ARROW"}],"results":[{"command":"UP_ARROW","expectation":"role up","pass":true,"output":"mocked output for role up"}]}]},{"filepath":"tests/test-2.json","log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}],"results":[{"testId":2,"capabilities":{"browserName":"mock","browserVersion":"1.0","atName":"mock","atVersion":"1.0","platformName":"mock"},"commands":[{"command":"UP_ARROW,DOWN_ARROW","output":"mocked output for UP_ARROW,DOWN_ARROW"}],"results":[{"command":"UP_ARROW,DOWN_ARROW","expectation":"role down","pass":true,"output":"mocked output for role down"}]}]}],"log":[{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."},{"data":{"type":"openPage","date":"2000-01-01T12:00:00.000Z","url":"http://localhost:8888/static/reference/index.html"},"text":"Open page: 'http://localhost:8888/static/reference/index.html'."}]}␊ `, } diff --git a/src/host/tests/snapshots/cli-run-plan.js.snap b/src/host/tests/snapshots/cli-run-plan.js.snap index b1aa412..5a15f78 100644 Binary files a/src/host/tests/snapshots/cli-run-plan.js.snap and b/src/host/tests/snapshots/cli-run-plan.js.snap differ diff --git a/src/host/tests/snapshots/messages.js.md b/src/host/tests/snapshots/messages.js.md index cf344b8..9b133cd 100644 --- a/src/host/tests/snapshots/messages.js.md +++ b/src/host/tests/snapshots/messages.js.md @@ -115,56 +115,9 @@ Generated by [AVA](https://avajs.dev). > Snapshot 11 - { - data: { - type: 'startAgent', - }, - text: 'Starting test agent.', - } - -> Snapshot 12 - - { - data: { - protocol: 'fork', - type: 'agentProtocol', - }, - text: 'Agent running with protocol \'fork\'.', - } - -> Snapshot 13 - - { - data: { - type: 'stopAgent', - }, - text: 'Stopping test agent.', - } - -> Snapshot 14 - - { - data: { - text: 'Starting test.', - type: 'agentLog', - }, - text: '[Agent]: Starting test.', - } - -> Snapshot 15 - - { - data: { - type: 'agentCrashed', - }, - text: 'Agent crashed.', - } - -> Snapshot 16 - { data: { type: 'startTest', }, - text: 'Starting test.', + text: 'Starting test #undefined \'undefined\'.', } diff --git a/src/host/tests/snapshots/messages.js.snap b/src/host/tests/snapshots/messages.js.snap index 7a44d39..ac32f57 100644 Binary files a/src/host/tests/snapshots/messages.js.snap and b/src/host/tests/snapshots/messages.js.snap differ diff --git a/src/host/types.js b/src/host/types.js index da80b73..0b8fba3 100644 --- a/src/host/types.js +++ b/src/host/types.js @@ -1,7 +1,7 @@ /// /// /// -/// +/// /** * @namespace AriaATCIHost @@ -15,17 +15,20 @@ * | 'planRead' * | 'serverListening' * | 'stopServer' + * | 'stopDrivers' * | 'addServerDirectory' * | 'removeServerDirectory' * | 'serverLog' - * | 'startAgent' - * | 'agentProtocol' - * | 'stopAgent' - * | 'agentLog' - * | 'agentCrashed' * | 'startTest' * | 'reportingError' * | 'testError' + * | 'atDriverComms' + * | 'openPage' + * | 'pressKeys' + * | 'speechEvent' + * | 'invalidKeys' + * | 'noRunTestSetup' + * | 'capabilities' * } AriaATCIHost.HostLogType */ @@ -34,7 +37,9 @@ */ /** - * @typedef {AriaATCIAgent.Log} AriaATCIHost.AgentLog + * @typedef AriaATCIHost.Logger + * @property {AriaATCIHost.Log} log + * @property {import("events").EventEmitter} emitter */ /** @@ -71,14 +76,6 @@ * @property {AriaATCIShared.BaseURL} baseUrl */ -/** - * @typedef AriaATCIHost.Agent - * @property {function(AriaATCIData.Test): Promise} run - * @property {function(): AsyncGenerator} logs - * @property {function(AriaATCIAgent.CliOptions): Promise} start - * @property {function(): Promise} stop - */ - /** * @callback AriaATCIHost.EmitPlanResults * @param {AriaATCIHost.TestPlan} plan diff --git a/src/agent/at-driver.js b/src/runner/at-driver.js similarity index 93% rename from src/agent/at-driver.js rename to src/runner/at-driver.js index 51cc518..7e5161c 100644 --- a/src/agent/at-driver.js +++ b/src/runner/at-driver.js @@ -1,7 +1,7 @@ import ws from 'ws'; import { iterateEmitter } from '../shared/iterate-emitter.js'; -import { AgentMessage } from './messages.js'; +import { RunnerMessage } from './messages.js'; /** * @param {object} options @@ -10,7 +10,7 @@ import { AgentMessage } from './messages.js'; * @param {string} [options.url.pathname] * @param {number | string} [options.url.port] * @param {Promise} [options.abortSignal] - * @param {AriaATCIAgent.Log} [options.log] + * @param {AriaATCIHost.Log} [options.log] * @returns {Promise} */ export async function createATDriver({ @@ -20,7 +20,7 @@ export async function createATDriver({ } = {}) { if (!abortSignal) process.exit(1); const url = `ws://${hostname}:${port}${pathname}`; - log(AgentMessage.AT_DRIVER_COMMS, { direction: 'connect', message: url }); + log(RunnerMessage.AT_DRIVER_COMMS, { direction: 'connect', message: url }); const socket = new ws(url); const driver = new ATDriver({ socket, log }); await driver.ready; @@ -47,7 +47,7 @@ export class ATDriver { this.closed = new Promise(resolve => socket.once('close', () => { this.hasClosed = true; - this.log(AgentMessage.AT_DRIVER_COMMS, { direction: 'closed' }); + this.log(RunnerMessage.AT_DRIVER_COMMS, { direction: 'closed' }); resolve(); }) ); @@ -60,7 +60,7 @@ export class ATDriver { } async quit() { - this.log(AgentMessage.AT_DRIVER_COMMS, { direction: 'close' }); + this.log(RunnerMessage.AT_DRIVER_COMMS, { direction: 'close' }); this.socket.close(); await this.closed; } @@ -69,7 +69,7 @@ export class ATDriver { if (this.hasClosed) throw new Error('AT-Driver connection unexpectedly closed'); for await (const rawMessage of iterateEmitter(this.socket, 'message', 'close', 'error')) { const message = rawMessage.toString(); - this.log(AgentMessage.AT_DRIVER_COMMS, { direction: 'inbound', message }); + this.log(RunnerMessage.AT_DRIVER_COMMS, { direction: 'inbound', message }); yield JSON.parse(message); } } @@ -77,7 +77,7 @@ export class ATDriver { async _send(command) { const id = this._nextId++; const rawMessage = JSON.stringify({ id, ...command }); - this.log(AgentMessage.AT_DRIVER_COMMS, { direction: 'outbound', message: rawMessage }); + this.log(RunnerMessage.AT_DRIVER_COMMS, { direction: 'outbound', message: rawMessage }); await new Promise((resolve, reject) => { this.socket.send(rawMessage, error => { if (error) reject(error); diff --git a/src/agent/browser-driver/create-safari-apple-script-driver.js b/src/runner/browser-driver/create-safari-apple-script-driver.js similarity index 100% rename from src/agent/browser-driver/create-safari-apple-script-driver.js rename to src/runner/browser-driver/create-safari-apple-script-driver.js diff --git a/src/agent/browser-driver/create-web-driver.js b/src/runner/browser-driver/create-web-driver.js similarity index 100% rename from src/agent/browser-driver/create-web-driver.js rename to src/runner/browser-driver/create-web-driver.js diff --git a/src/agent/browser-driver/create.js b/src/runner/browser-driver/create.js similarity index 92% rename from src/agent/browser-driver/create.js rename to src/runner/browser-driver/create.js index 0192a55..28944c6 100644 --- a/src/agent/browser-driver/create.js +++ b/src/runner/browser-driver/create.js @@ -4,7 +4,7 @@ import createSafariAppleScriptDriver from './create-safari-apple-script-driver.j /** * @param {object} options * @param {{toString: function(): string}} options.url - * @param {AriaATCIAgent.Browser} [options.browser] + * @param {AriaATCIRunner.Browser} [options.browser] * @param {Promise} options.abortSignal * @param {AriaATCIShared.timesOption} options.timesOption * diff --git a/src/agent/create-test-runner.js b/src/runner/create-test-runner.js similarity index 63% rename from src/agent/create-test-runner.js rename to src/runner/create-test-runner.js index 4c89984..6c07b7a 100644 --- a/src/agent/create-test-runner.js +++ b/src/runner/create-test-runner.js @@ -9,45 +9,43 @@ import { MockTestRunner } from './mock-test-runner.js'; import { DriverTestRunner } from './driver-test-runner.js'; import { createBrowserDriver } from './browser-driver/create.js'; import { createATDriver } from './at-driver.js'; -import { AgentMessage } from './messages.js'; /** * @param {object} options - * @param {Promise} options.abortSignal resolves when runner should stop * @param {{hostname: string, port: number | string, pathname: string}} options.atDriverUrl - * @param {AriaATCIShared.BaseURL} options.baseUrl - * @param {AriaATCIAgent.Log} options.log - * @param {AriaATCIAgent.MockOptions} [options.mock] - * @param {AriaATCIAgent.Browser} [options.webDriverBrowser] + * @param {URL} options.baseUrl + * @param {AriaATCIHost.Log} options.log + * @param {Promise} options.abortSignal + * @param {boolean} [options.mock] + * @param {AriaATCIRunner.Browser} [options.webDriverBrowser] * @param {AriaATCIShared.timesOption} options.timesOption * @param {{toString: function(): string}} options.webDriverUrl - * @returns {Promise} + * @returns {Promise} */ export async function createRunner(options) { - if (!options.abortSignal) { - throw new Error('createRunner requires abortSignal option.'); - } + const { abortSignal, log, timesOption } = options; + if (options.mock) { - return new MockTestRunner({ mock: options.mock, ...options }); + return new MockTestRunner(options); } await new Promise(resolve => setTimeout(resolve, 1000)); - const { timesOption } = options; + const [browserDriver, atDriver] = await Promise.all([ createBrowserDriver({ url: options.webDriverUrl, browser: options.webDriverBrowser, - abortSignal: options.abortSignal, + abortSignal, timesOption, }).catch(cause => { throw new Error('Error initializing browser driver', { cause }); }), createATDriver({ url: options.atDriverUrl, - abortSignal: options.abortSignal, - log: options.log, + abortSignal, + log, }).catch(cause => { throw new Error('Error connecting to at-driver', { cause }); }), ]); - return new DriverTestRunner({ ...options, browserDriver, atDriver, timesOption }); + return new DriverTestRunner({ ...options, browserDriver, atDriver }); } diff --git a/src/agent/driver-test-runner.js b/src/runner/driver-test-runner.js similarity index 91% rename from src/agent/driver-test-runner.js rename to src/runner/driver-test-runner.js index 2489f34..c438a1d 100644 --- a/src/agent/driver-test-runner.js +++ b/src/runner/driver-test-runner.js @@ -5,7 +5,7 @@ import { startJob } from '../shared/job.js'; import { ATDriver, ATKey, webDriverCodePoints } from './at-driver.js'; -import { AgentMessage } from './messages.js'; +import { RunnerMessage } from './messages.js'; /** * @module agent @@ -14,8 +14,8 @@ import { AgentMessage } from './messages.js'; export class DriverTestRunner { /** * @param {object} options - * @param {AriaATCIShared.BaseURL} options.baseUrl - * @param {AriaATCIAgent.Log} options.log + * @param {URL} options.baseUrl + * @param {AriaATCIHost.Log} options.log * @param {BrowserDriver} options.browserDriver * @param {ATDriver} options.atDriver * @param {AriaATCIShared.timesOption} options.timesOption @@ -41,7 +41,7 @@ export class DriverTestRunner { * @param {string} options.referencePage */ async openPage({ url, referencePage }) { - await this.log(AgentMessage.OPEN_PAGE, { url }); + await this.log(RunnerMessage.OPEN_PAGE, { url }); await this.browserDriver.navigate(url.toString()); await this.browserDriver.documentReady(); @@ -52,20 +52,20 @@ export class DriverTestRunner { this.timesOption.testSetup ); } catch ({}) { - await this.log(AgentMessage.NO_RUN_TEST_SETUP, { referencePage }); + await this.log(RunnerMessage.NO_RUN_TEST_SETUP, { referencePage }); } } /** - * @param {import('./at-driver').ATKeySequence} sequence + * @param {import('./at-driver.js').ATKeySequence} sequence */ async sendKeys(sequence) { - await this.log(AgentMessage.PRESS_KEYS, { keys: sequence }); + await this.log(RunnerMessage.PRESS_KEYS, { keys: sequence }); await this.atDriver.sendKeys(sequence); } /** - * @param {import('./at-driver').ATKeySequence} sequence + * @param {import('./at-driver.js').ATKeySequence} sequence * @param {string} desiredResponse */ async pressKeysToToggleSetting(sequence, desiredResponse) { @@ -181,11 +181,11 @@ export class DriverTestRunner { */ async run(test) { const capabilities = await this.collectedCapabilities; - await this.log(AgentMessage.CAPABILITIES, { capabilities }); + await this.log(RunnerMessage.CAPABILITIES, { capabilities }); - await this.log(AgentMessage.START_TEST, { id: test.info.testId, title: test.info.task }); + await this.log(RunnerMessage.START_TEST, { id: test.info.testId, title: test.info.task }); - await this.log(AgentMessage.OPEN_PAGE, { url: 'about:blank' }); + await this.log(RunnerMessage.OPEN_PAGE, { url: 'about:blank' }); await this.browserDriver.navigate('about:blank'); const commandsOutput = []; @@ -215,7 +215,7 @@ export class DriverTestRunner { ); await this._collectSpeech(this.timesOption.afterNav, async () => { - await this.log(AgentMessage.OPEN_PAGE, { url: 'about:blank' }); + await this.log(RunnerMessage.OPEN_PAGE, { url: 'about:blank' }); await this.browserDriver.navigate('about:blank'); }); @@ -232,7 +232,7 @@ export class DriverTestRunner { }); } } else { - await this.log(AgentMessage.INVALID_KEYS, { command, errors }); + await this.log(RunnerMessage.INVALID_KEYS, { command, errors }); commandsOutput.push({ command: command.id, @@ -270,7 +270,7 @@ export class DriverTestRunner { const speechJob = startJob(async signal => { for await (const speech of signal.cancelable(this.atDriver.speeches())) { spoken.push(speech); - this.log(AgentMessage.SPEECH_EVENT, { spokenText: speech }); + this.log(RunnerMessage.SPEECH_EVENT, { spokenText: speech }); } }); @@ -288,10 +288,8 @@ export class DriverTestRunner { } _appendBaseUrl(pathname) { - // protocol ends with a ':' and pathname starts with a '/' - const base = `${this.baseUrl.protocol}//${this.baseUrl.hostname}:${this.baseUrl.port}${this.baseUrl.pathname}`; const newPath = `${this.baseUrl.pathname ? `${this.baseUrl.pathname}/` : ''}${pathname}`; - return new URL(newPath, base); + return new URL(newPath, this.baseUrl.toString()); } } diff --git a/src/runner/messages.js b/src/runner/messages.js new file mode 100644 index 0000000..a015664 --- /dev/null +++ b/src/runner/messages.js @@ -0,0 +1,47 @@ +/// +/// + +/** + * @module runner + */ + +import { createSharedLogger } from '../shared/messages.js'; + +/** @enum {AriaATCIRunner.Message} */ +export const RunnerMessage = { + /** @type {'startTest'} */ + START_TEST: 'startTest', + /** @type {'openPage'} */ + OPEN_PAGE: 'openPage', + /** @type {'invalidKeys'} */ + INVALID_KEYS: 'invalidKeys', + /** @type {'pressKeys'} */ + PRESS_KEYS: 'pressKeys', + /** @type {'speechEvent'} */ + SPEECH_EVENT: 'speechEvent', + /** @type {'noRunTestSetup'} */ + NO_RUN_TEST_SETUP: 'noRunTestSetup', + /** @type {'atDriverComms'} */ + AT_DRIVER_COMMS: 'atDriverComms', + /** @type {'capabilities'} */ + CAPABILITIES: 'capabilities', +}; + +export const RUNNER_TEMPLATES = { + [RunnerMessage.START_TEST]: ({ id, title }) => `Starting test #${id} '${title}'.`, + [RunnerMessage.OPEN_PAGE]: ({ url }) => `Open page: '${url}'.`, + [RunnerMessage.INVALID_KEYS]: ({ command, errors }) => + `Keys in '${command.id}' have issues:\n${errors.map(error => `- ${error}`).join('\n')}`, + [RunnerMessage.PRESS_KEYS]: ({ keys }) => `Press keys: '${keys.toString()}'.`, + [RunnerMessage.SPEECH_EVENT]: ({ spokenText }) => `Speech event: '${spokenText}'.`, + [RunnerMessage.NO_RUN_TEST_SETUP]: ({ referencePage }) => + `Test reference, ${referencePage}, does not have a Run Test Setup button.`, + [RunnerMessage.AT_DRIVER_COMMS]: ({ direction, message }) => + `AT-Driver: ${direction}: ${message}`, + [RunnerMessage.CAPABILITIES]: ({ capabilities }) => + `Capabilities: ${JSON.stringify(capabilities)}`, +}; + +export function createRunnerLogger(messages = RUNNER_TEMPLATES) { + return createSharedLogger(messages); +} diff --git a/src/agent/mock-test-runner.js b/src/runner/mock-test-runner.js similarity index 67% rename from src/agent/mock-test-runner.js rename to src/runner/mock-test-runner.js index e13ccb8..35af8cb 100644 --- a/src/agent/mock-test-runner.js +++ b/src/runner/mock-test-runner.js @@ -7,49 +7,45 @@ */ import { request } from 'http'; -import { AgentMessage } from './messages.js'; +import { RunnerMessage } from './messages.js'; import { validateKeysFromCommand } from './driver-test-runner.js'; /** - * @implements {AriaATCIAgent.TestRunner} + * @implements {AriaATCIRunner.TestRunner} */ export class MockTestRunner { /** * @param {object} options - * @param {AriaATCIShared.BaseURL} options.baseUrl - * @param {AriaATCIAgent.Log} options.log - * @param {AriaATCIAgent.MockOptions} options.mock + * @param {URL} options.baseUrl + * @param {AriaATCIHost.Log} options.log */ - constructor({ baseUrl, log, mock: config }) { + constructor({ baseUrl, log }) { this.baseUrl = baseUrl; this.log = log; - this.config = config; } async openPage(url) { - if (this.config.openPage === 'request') { - await new Promise((resolve, reject) => - request(url.toString(), res => { - try { - res - .on('data', () => {}) - .on('error', reject) - .setEncoding('utf8') - .on('end', () => { - res.statusCode < 400 - ? resolve() - : reject(new Error(`request returned ${res.statusCode}`)); - }); - } catch (e) { - reject(e); - } - }) - .on('error', reject) - .end() - ); - } + await new Promise((resolve, reject) => + request(url.toString(), res => { + try { + res + .on('data', () => {}) + .on('error', reject) + .setEncoding('utf8') + .on('end', () => { + res.statusCode < 400 + ? resolve() + : reject(new Error(`request returned ${res.statusCode}`)); + }); + } catch (e) { + reject(e); + } + }) + .on('error', reject) + .end() + ); - this.log(AgentMessage.OPEN_PAGE, { url }); + this.log(RunnerMessage.OPEN_PAGE, { url }); } /** @@ -64,11 +60,10 @@ export class MockTestRunner { * @param {AriaATCIData.CollectedTest} task */ async run(task) { - const base = `${this.baseUrl.protocol}//${this.baseUrl.hostname}:${this.baseUrl.port}${this.baseUrl.pathname}`; await this.openPage( new URL( `${this.baseUrl.pathname ? `${this.baseUrl.pathname}/` : ''}${task.target.referencePage}`, - base + this.baseUrl.toString() ) ); @@ -95,7 +90,7 @@ export class MockTestRunner { }); } } else { - await this.log(AgentMessage.INVALID_KEYS, { command, errors }); + await this.log(RunnerMessage.INVALID_KEYS, { command, errors }); commandsOutput.push({ command: command.id, diff --git a/src/agent/types.js b/src/runner/types.js similarity index 67% rename from src/agent/types.js rename to src/runner/types.js index d9c36ef..07e0ebc 100644 --- a/src/agent/types.js +++ b/src/runner/types.js @@ -1,7 +1,7 @@ /// /// -/** @namespace AriaATCIAgent */ +/** @namespace AriaATCIRunner */ /** * @typedef {'start' @@ -15,47 +15,41 @@ * | 'noRunTestSetup' * | 'atDriverComms' * | 'capabilities' - * } AriaATCIAgent.Message + * } AriaATCIRunner.Message */ /** - * @typedef {AriaATCIShared.Log} AriaATCIAgent.Log + * @typedef {AriaATCIShared.Log} AriaATCIRunner.Log */ /** - * @typedef {AsyncIterable} AriaATCIAgent.TestIterable + * @typedef {AsyncIterable} AriaATCIRunner.TestIterable */ /** - * @typedef AriaATCIAgent.TestRunner + * @typedef AriaATCIRunner.TestRunner * @property {function(AriaATCIData.Test): Promise} run run a test */ /** - * @callback AriaATCIAgent.ReportResult + * @callback AriaATCIRunner.ReportResult * @param {AriaATCIData.TestResult} result * @returns {Promise} */ /** - * @typedef AriaATCIAgent.MockOptions - * @property {'request' | 'skip'} [openPage] + * @typedef {'chrome' | 'firefox' | 'safari'} AriaATCIRunner.Browser */ /** - * @typedef {'chrome' | 'firefox' | 'safari'} AriaATCIAgent.Browser - */ - -/** - * @typedef AriaATCIAgent.CliOptions + * @typedef AriaATCIRunner.CliOptions * @property {boolean} [debug] * @property {boolean} [quiet] - * @property {AriaATCIAgent.Message[]} [verbose] + * @property {AriaATCIRunner.Message[]} [verbose] * @property {AriaATCIShared.BaseURL} [referenceBaseUrl] * @property {boolean} [mock] - * @property {'request' | 'skip'} [mockOpenPage] * @property {AriaATCIShared.BaseURL} [webDriverUrl] - * @property {AriaATCIAgent.Browser} [webDriverBrowser] + * @property {AriaATCIRunner.Browser} [webDriverBrowser] * @property {AriaATCIShared.BaseURL} [atDriverUrl] * @property {AriaATCIShared.timesOption} [timesOption] */